first commit

sources/source_types/AbstractSource.py

@@ -0,0 +1,37 @@
+from abc import ABC, abstractmethod
+
+from pydantic import Extra, BaseModel
+
+from sources.file_types.AbstractParser import AbstractParser
+
+
+class AbstractSource(BaseModel, ABC):
+    file_hash: str = None
+    source_name: str = None
+    file_parser: AbstractParser = None
+
+    class Config:
+        extra = Extra.allow
+
+    def __init__(self, source_name, **kwargs):
+        super().__init__(**kwargs)
+        if not source_name:
+            raise Exception('You have to add "source_name" to your config!')
+        self.source_name = source_name
+
+    @abstractmethod
+    def check_is_update_needed(self):
+        pass
+
+    @abstractmethod
+    def parse_data(self):
+        pass
+
+    def is_parser_needed(self):
+        return False
+
+    def active_file_hash(self):
+        return self.file_hash
+
+    def after_save(self):
+        pass

sources/source_types/FtpSource.py

@@ -0,0 +1,58 @@
+import os
+
+from components.FileComponent import FileComponent
+from components.FtpDownloader import FtpDownloader
+from sources.source_types.AbstractSource import AbstractSource
+
+
+class FtpSource(AbstractSource):
+
+    # config
+    ftp_url: str
+    ftp_login: str
+    ftp_password: str
+    file_regex: str
+    local_files_path: str
+
+    # additional params
+    source_files: list = []
+    files_for_update: list = []
+    local_files: list = []
+
+    def __init__(self, source_name, **kwargs):
+        super().__init__(source_name, **kwargs)
+        self.fileComponent = FileComponent(local_files_path=self.local_files_path)
+        self.local_files = self.fileComponent.get_parsed_files(source_name)
+        self.ftpClient = FtpDownloader(url=self.ftp_url, login=self.ftp_login, password=self.ftp_password)
+        self.source_files = self.ftpClient.get_files_with_checksum('/', self.file_regex)
+        self.__files_for_update()
+
+    def __files_for_update(self):
+        local_hashes = [f.get('hash') for f in self.local_files]
+        self.files_for_update = [f for f in self.source_files if f.get('hash') not in local_hashes]
+        return self.files_for_update
+
+    def is_parser_needed(self):
+        return True
+
+    def download_file(self, file):
+        if not os.path.exists(self.local_files_path):
+            os.makedirs(self.local_files_path)
+        self.ftpClient.download_file(file, self.local_files_path + os.path.basename(file))
+
+    def check_is_update_needed(self):
+        return len(self.files_for_update) > 0
+
+    def parse_data(self):
+        for f in self.files_for_update:
+            self.download_file(f.get('file'))
+            self.file_hash = f.get('hash')
+            file_path = self.local_files_path + f.get('file')
+            data = self.file_parser.parse(file_path)
+            for line in data:
+                yield line
+
+    def after_save(self):
+        for f in self.files_for_update:
+            self.fileComponent.add_parsed_file(self.source_name, f.get('file'), f.get('hash'), f.get('updated_at'))
+            self.fileComponent.remove_file(self.local_files_path + f.get('file'))