first commit

components/FileComponent.py

@@ -0,0 +1,21 @@
+import os
+
+from pydantic import BaseModel
+
+from models.File import File
+
+
+class FileComponent(BaseModel):
+
+    local_files_path: str
+
+    def get_parsed_files(self, source_name):
+        files = [{"file": self.local_files_path + f.file_path, "hash": f.file_hash}
+                            for f in File.objects(source=source_name)]
+        return files
+
+    def add_parsed_file(self, source, path, hash, time):
+        return File(source=source, file_path=path, file_hash=hash, file_updated_at=time).save()
+
+    def remove_file(self, file):
+        os.remove(file)

components/FtpDownloader.py

@@ -0,0 +1,65 @@
+import hashlib
+
+from pydantic import BaseModel, Extra
+from ftplib import FTP
+from urllib.parse import urlparse
+
+class FtpDownloader(BaseModel):
+    url: str
+    login = ''
+    password = ''
+
+    class Config:
+        extra = Extra.allow
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.__connect()
+
+    def __del__(self):
+        self.ftp.close()
+
+    def __connect(self):
+        parsed_uri = urlparse(self.url)
+        self.ftp = FTP(parsed_uri.netloc)
+        self.ftp.login(self.login, self.password)
+        if parsed_uri.path:
+            self.ftp.cwd(parsed_uri.path)
+
+    def get_files_with_checksum(self, path, filetypeRegex = None):
+        if filetypeRegex:
+            path += filetypeRegex
+        files = self.ftp.nlst(path)
+
+        file_list = []
+
+        for file_path in files:
+            m = hashlib.sha1()
+            self.ftp.retrbinary('RETR %s' % file_path, m.update)
+            updated_at = self.ftp.voidcmd("MDTM " + file_path)[4:].strip()
+            file_dict = {'file': file_path, 'hash': m.hexdigest(), 'updated_at': updated_at}
+            file_list.append(file_dict)
+            print(file_dict)
+
+        file_list = sorted(file_list, key=lambda d: d['updated_at'])
+        return file_list
+
+    def download_file(self, origin_file, to):
+        m = hashlib.sha1()
+        self.ftp.retrbinary('RETR %s' % origin_file, m.update)
+        checksum = m.hexdigest()
+
+        # Write file in binary mode
+        with open(to, "wb") as file:
+            self.ftp.retrbinary(f"RETR {origin_file}", file.write)
+
+        file.close()
+        file = open(to, "rb")
+        local_checksum = hashlib.sha1(file.read()).hexdigest()
+
+        if checksum and local_checksum != checksum:
+            raise BaseException(f"Wrong checksum for file: {origin_file}")
+
+        file.close()
+        print("file downloaded " + origin_file)
+        return True

components/SenderComponent.py

@@ -0,0 +1,23 @@
+import json
+import requests
+
+from pydantic import BaseModel
+
+
+class SenderComponent(BaseModel):
+    api_url: str
+    api_key: str
+
+    def __generate_key(self):
+        return 'Bearer {key}'.format(key=self.api_key)
+
+    def __headers(self):
+        return {
+            'Authorization': self.__generate_key(),
+            'Content-type': 'application/json',
+        }
+
+    def send_data(self, data):
+        headers = self.__headers()
+        response = requests.post(self.api_url + 'data', data=json.dumps(data), headers=headers)
+        return response.status_code == 200