first commit

components/FtpDownloader.py

@@ -0,0 +1,65 @@
+import hashlib
+
+from pydantic import BaseModel, Extra
+from ftplib import FTP
+from urllib.parse import urlparse
+
+class FtpDownloader(BaseModel):
+    url: str
+    login = ''
+    password = ''
+
+    class Config:
+        extra = Extra.allow
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.__connect()
+
+    def __del__(self):
+        self.ftp.close()
+
+    def __connect(self):
+        parsed_uri = urlparse(self.url)
+        self.ftp = FTP(parsed_uri.netloc)
+        self.ftp.login(self.login, self.password)
+        if parsed_uri.path:
+            self.ftp.cwd(parsed_uri.path)
+
+    def get_files_with_checksum(self, path, filetypeRegex = None):
+        if filetypeRegex:
+            path += filetypeRegex
+        files = self.ftp.nlst(path)
+
+        file_list = []
+
+        for file_path in files:
+            m = hashlib.sha1()
+            self.ftp.retrbinary('RETR %s' % file_path, m.update)
+            updated_at = self.ftp.voidcmd("MDTM " + file_path)[4:].strip()
+            file_dict = {'file': file_path, 'hash': m.hexdigest(), 'updated_at': updated_at}
+            file_list.append(file_dict)
+            print(file_dict)
+
+        file_list = sorted(file_list, key=lambda d: d['updated_at'])
+        return file_list
+
+    def download_file(self, origin_file, to):
+        m = hashlib.sha1()
+        self.ftp.retrbinary('RETR %s' % origin_file, m.update)
+        checksum = m.hexdigest()
+
+        # Write file in binary mode
+        with open(to, "wb") as file:
+            self.ftp.retrbinary(f"RETR {origin_file}", file.write)
+
+        file.close()
+        file = open(to, "rb")
+        local_checksum = hashlib.sha1(file.read()).hexdigest()
+
+        if checksum and local_checksum != checksum:
+            raise BaseException(f"Wrong checksum for file: {origin_file}")
+
+        file.close()
+        print("file downloaded " + origin_file)
+        return True