From 2bde5c709a86c77ce13d8be78bbd2eceb65cbfe3 Mon Sep 17 00:00:00 2001 From: ekazak Date: Wed, 14 Dec 2022 12:36:37 +0100 Subject: [PATCH 01/35] first commit --- Dockerfile | 11 +++ README.md | 76 ++++++++++++++++- components/FileComponent.py | 21 +++++ components/FtpDownloader.py | 65 +++++++++++++++ components/SenderComponent.py | 23 ++++++ configs/application_credentials.json | 6 ++ configs/config.py | 64 +++++++++++++++ configs/db.json | 6 ++ configs/main.json | 3 + configs/sources.json | 0 models/File.py | 11 +++ models/MetaData.py | 9 +++ sources/BaseSource.py | 108 +++++++++++++++++++++++++ sources/file_types/AbstractParser.py | 13 +++ sources/file_types/CsvParser.py | 12 +++ sources/source_types/AbstractSource.py | 37 +++++++++ sources/source_types/FtpSource.py | 58 +++++++++++++ update.py | 49 +++++++++++ 18 files changed, 570 insertions(+), 2 deletions(-) create mode 100644 Dockerfile create mode 100644 components/FileComponent.py create mode 100644 components/FtpDownloader.py create mode 100644 components/SenderComponent.py create mode 100644 configs/application_credentials.json create mode 100644 configs/config.py create mode 100644 configs/db.json create mode 100644 configs/main.json create mode 100644 configs/sources.json create mode 100644 models/File.py create mode 100644 models/MetaData.py create mode 100644 sources/BaseSource.py create mode 100644 sources/file_types/AbstractParser.py create mode 100644 sources/file_types/CsvParser.py create mode 100644 sources/source_types/AbstractSource.py create mode 100644 sources/source_types/FtpSource.py create mode 100644 update.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6339c6c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.12-rc-slim + +WORKDIR /app + +RUN pip install pymongo \ + dynaconf \ + pydantic \ + pymysql \ + mongoengine \ + dict_hash \ + requests diff --git a/README.md b/README.md index 10199ce..028b29c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,74 @@ -# LiveCarta-microservices -microservices +# Book Meta Data Parser + +Microservice which solves only one issue – parse book meta data from our publishers. Not depends on what format publisher stores this data, the service must grub this information and send an array of data to the main application without any formatting. The main idea is to add components for parsing different formats and have the ability to add publishers just by updating config files. + + +## Version 1.0 + +Added two components for working with CSV and FTP. + + +## Tech Stack + + • Docker + • Python 3.11 + • MongoDb 6.0.2 + • Dynaconf + • Pydantic + • MongoEngine + + +## Folder structure + + + • app + ◦ components + ◦ configs + ▪ application_credentials.json – keys and url for connection to our main app + ▪ db.json – creds for service db + ▪ main.json – main config + ▪ sources.json – list of sources with components that they use + ◦ models + ◦ sources + ▪ file_types + ▪ source_types + + +## Sources configuration + +To configure a new source you need to update source config by adding the params below: + + • source_name + • source //with neccesary params for component + • parser_type //with neccesary params for component + +Example for CSV files from FTP: +```json + { + "sources": { + "McGrawHill": { + "source_name": "McGrawHill", + "source": { + "type": "ftp", + "ftp_url": "127.0.0.1", + "ftp_login": "frp_login", + "ftp_password": "frp_pass", + "local_files_path": "/app/files/McGrawHill/", + "file_regex": "*.csv" + }, + "parser_type": { + "format": "csv" + } + } + } + } +``` + +Each source parser starts by crontab by command + +`python update.py {source_name}` + + +To see list of source types use command + +`python update.py -h` diff --git a/components/FileComponent.py b/components/FileComponent.py new file mode 100644 index 0000000..d70a00a --- /dev/null +++ b/components/FileComponent.py @@ -0,0 +1,21 @@ +import os + +from pydantic import BaseModel + +from models.File import File + + +class FileComponent(BaseModel): + + local_files_path: str + + def get_parsed_files(self, source_name): + files = [{"file": self.local_files_path + f.file_path, "hash": f.file_hash} + for f in File.objects(source=source_name)] + return files + + def add_parsed_file(self, source, path, hash, time): + return File(source=source, file_path=path, file_hash=hash, file_updated_at=time).save() + + def remove_file(self, file): + os.remove(file) \ No newline at end of file diff --git a/components/FtpDownloader.py b/components/FtpDownloader.py new file mode 100644 index 0000000..5eb62d2 --- /dev/null +++ b/components/FtpDownloader.py @@ -0,0 +1,65 @@ +import hashlib + +from pydantic import BaseModel, Extra +from ftplib import FTP +from urllib.parse import urlparse + +class FtpDownloader(BaseModel): + url: str + login = '' + password = '' + + class Config: + extra = Extra.allow + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.__connect() + + def __del__(self): + self.ftp.close() + + def __connect(self): + parsed_uri = urlparse(self.url) + self.ftp = FTP(parsed_uri.netloc) + self.ftp.login(self.login, self.password) + if parsed_uri.path: + self.ftp.cwd(parsed_uri.path) + + def get_files_with_checksum(self, path, filetypeRegex = None): + if filetypeRegex: + path += filetypeRegex + files = self.ftp.nlst(path) + + file_list = [] + + for file_path in files: + m = hashlib.sha1() + self.ftp.retrbinary('RETR %s' % file_path, m.update) + updated_at = self.ftp.voidcmd("MDTM " + file_path)[4:].strip() + file_dict = {'file': file_path, 'hash': m.hexdigest(), 'updated_at': updated_at} + file_list.append(file_dict) + print(file_dict) + + file_list = sorted(file_list, key=lambda d: d['updated_at']) + return file_list + + def download_file(self, origin_file, to): + m = hashlib.sha1() + self.ftp.retrbinary('RETR %s' % origin_file, m.update) + checksum = m.hexdigest() + + # Write file in binary mode + with open(to, "wb") as file: + self.ftp.retrbinary(f"RETR {origin_file}", file.write) + + file.close() + file = open(to, "rb") + local_checksum = hashlib.sha1(file.read()).hexdigest() + + if checksum and local_checksum != checksum: + raise BaseException(f"Wrong checksum for file: {origin_file}") + + file.close() + print("file downloaded " + origin_file) + return True \ No newline at end of file diff --git a/components/SenderComponent.py b/components/SenderComponent.py new file mode 100644 index 0000000..c0d414d --- /dev/null +++ b/components/SenderComponent.py @@ -0,0 +1,23 @@ +import json +import requests + +from pydantic import BaseModel + + +class SenderComponent(BaseModel): + api_url: str + api_key: str + + def __generate_key(self): + return 'Bearer {key}'.format(key=self.api_key) + + def __headers(self): + return { + 'Authorization': self.__generate_key(), + 'Content-type': 'application/json', + } + + def send_data(self, data): + headers = self.__headers() + response = requests.post(self.api_url + 'data', data=json.dumps(data), headers=headers) + return response.status_code == 200 diff --git a/configs/application_credentials.json b/configs/application_credentials.json new file mode 100644 index 0000000..d3a1be3 --- /dev/null +++ b/configs/application_credentials.json @@ -0,0 +1,6 @@ +{ + "application_credentials": { + "api_url":"http://app.livecarta.loc/meta/api/", + "api_key":"695e513c-xxxx-xxxx-a666-xxxxxxxxxx" + } +} \ No newline at end of file diff --git a/configs/config.py b/configs/config.py new file mode 100644 index 0000000..1b09ae5 --- /dev/null +++ b/configs/config.py @@ -0,0 +1,64 @@ +import json + +import requests +from dynaconf import Dynaconf, Validator +from mongoengine import connect + + +class AppConfig: + def __init__(self): + self.config = Dynaconf(settings_files=[ + "/app/configs/main.json", + "/app/configs/application_credentials.json", + "/app/configs/db.json", + "/app/configs/sources.json" + ]) + self.config.validators.register( + Validator('db', 'db.host', 'db.database', must_exist=True), + ) + self.config.validators.validate() + + creds = self.get_db_config() + connect( + db=creds.database, + host="mongodb://{host}:27017/{database}".format(host=creds.host, database=creds.database) + ) + + def get_bulk_insert_limit(self): + if not self.config.bulk_limit: + return 1 + else: + return self.config.bulk_limit + + def get_db_config(self): + return self.config.db + + def get_main_app_creds(self): + return self.config.application_credentials + + def get_source_by_name(self, name: str): + if name not in self.config.sources: + raise ValueError(f'"{name}" source not exists!') + return self.config.sources[name] + + def get_sources_list(self): + return self.config.sources.keys() + + def update_sources(self): + creds = self.get_main_app_creds() + headers = { + 'Content-type': 'application/json', + 'Authorization': 'Bearer {key}'.format(key=creds.api_key) + } + r = requests.get(creds.api_url + "sources", headers=headers) + if r.status_code != 200: + raise Exception('Bad app response') + + new_config = {"sources": {}} + for source in r.json(): + new_config["sources"][source["source_name"]] = source + + with open("./configs/sources.json", "w") as outfile: + outfile.write(json.dumps(new_config)) + +config = AppConfig() \ No newline at end of file diff --git a/configs/db.json b/configs/db.json new file mode 100644 index 0000000..2d3e8a6 --- /dev/null +++ b/configs/db.json @@ -0,0 +1,6 @@ +{ + "db": { + "host": "mongo_book_meta", + "database": "mongo_book_meta" + } +} \ No newline at end of file diff --git a/configs/main.json b/configs/main.json new file mode 100644 index 0000000..0b762b3 --- /dev/null +++ b/configs/main.json @@ -0,0 +1,3 @@ +{ + "bulk_limit": 100 +} \ No newline at end of file diff --git a/configs/sources.json b/configs/sources.json new file mode 100644 index 0000000..e69de29 diff --git a/models/File.py b/models/File.py new file mode 100644 index 0000000..37da8e1 --- /dev/null +++ b/models/File.py @@ -0,0 +1,11 @@ +from datetime import datetime +from mongoengine import * + + +class File(Document): + + file_hash = StringField(required=True, max_length=40, primary_key=True) + source = StringField(required=True, max_length=40) + file_path = StringField(required=True, max_length=255) + file_updated_at = IntField(required=False) + created_at = DateTimeField(default=datetime.utcnow) diff --git a/models/MetaData.py b/models/MetaData.py new file mode 100644 index 0000000..3381215 --- /dev/null +++ b/models/MetaData.py @@ -0,0 +1,9 @@ +from datetime import datetime +from mongoengine import * + + +class MetaData(Document): + + row_hash = StringField(required=True, max_length=40, primary_key=True) + file_hash = StringField(max_length=40) + created_at = DateTimeField(default=datetime.utcnow) \ No newline at end of file diff --git a/sources/BaseSource.py b/sources/BaseSource.py new file mode 100644 index 0000000..aac4d9d --- /dev/null +++ b/sources/BaseSource.py @@ -0,0 +1,108 @@ +import hashlib +import importlib +import json + +import mongoengine +from pydantic import BaseModel, Extra + +from components.SenderComponent import SenderComponent +from configs.config import config +from models.MetaData import MetaData +from sources.source_types.AbstractSource import AbstractSource + + +class BaseSource(BaseModel): + source_name: str + source: dict + parser_type: dict + + sourceObject: AbstractSource = None + senderComponent: SenderComponent = None + + class Config: + extra = Extra.allow + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.senderComponent = SenderComponent(**config.get_main_app_creds()) + + def get_source_object(config): + class_name = config.type.capitalize() + "Source" + module_name = 'sources.source_types.' + class_name + module = importlib.import_module(module_name) + module_class = getattr(module, class_name) + return module_class(source_name=self.source_name, **config) + + def get_parser(config): + class_name = config.format.capitalize() + "Parser" + module_name = 'sources.file_types.' + class_name + module = importlib.import_module(module_name) + module_class = getattr(module, class_name) + return module_class(**config) + + self.sourceObject = get_source_object(self.source) + + if (self.sourceObject.is_parser_needed()): + self.sourceObject.file_parser = get_parser(self.parser_type) + + def check_is_update_needed(self): + return self.sourceObject.check_is_update_needed() + + def check_is_new_row(self, hash): + try: + MetaData.objects().get(row_hash=hash) + return False + except MetaData.DoesNotExist: + return True + + def save_row_hash(self, row_hash, data, file_hash=None): # TODO: is origina_data needed? + MetaData(file_hash=file_hash, row_hash=row_hash, original_data=data).save() + + def __bulk_save(self, hashes, data): + if self.senderComponent.send_data({'data': data, 'source_name': self.source_name}): + instances = [MetaData(**data) for data in hashes] + try: + MetaData.objects.insert(instances, load_bulk=False) + except mongoengine.errors.BulkWriteError as error: + print(error) + self.__insert_row_by_row(instances) + else: + raise Exception("Error on saving data.") + + def __insert_row_by_row(self, instances): + for model in instances: + try: + model.save() + except mongoengine.errors.BulkWriteError: + print("Duplicate row with id: {id}".format(id=model.row_hash)) + + def update_data(self): + data_iterator = self.sourceObject.parse_data() + + inserted_rows = 0 + # try: + data_to_send = [] + hashes = [] + for row in data_iterator: + dhash = hashlib.sha1() + encoded = json.dumps(row, sort_keys=True).encode() + dhash.update(encoded) + row_hash = dhash.hexdigest() + if self.check_is_new_row(row_hash): + data_to_send.append(row) + hashes.append({"row_hash": row_hash, "file_hash": self.sourceObject.active_file_hash()}) + if len(hashes) >= config.get_bulk_insert_limit(): + self.__bulk_save(hashes, data_to_send) + inserted_rows += len(hashes) + data_to_send = [] + hashes = [] + + if len(hashes) > 0: + self.__bulk_save(hashes, data_to_send) + self.sourceObject.after_save() + # except Exception as inst: + # print('Undefined error! Data not updated') + # print(type(inst)) + # print(inst.args) + + return inserted_rows diff --git a/sources/file_types/AbstractParser.py b/sources/file_types/AbstractParser.py new file mode 100644 index 0000000..ce35862 --- /dev/null +++ b/sources/file_types/AbstractParser.py @@ -0,0 +1,13 @@ +from abc import ABC, abstractmethod +from pydantic import Extra, BaseModel + + +class AbstractParser(BaseModel, ABC): + + + class Config: + extra = Extra.allow + + @abstractmethod + def parse(self, source): + pass \ No newline at end of file diff --git a/sources/file_types/CsvParser.py b/sources/file_types/CsvParser.py new file mode 100644 index 0000000..8acefe3 --- /dev/null +++ b/sources/file_types/CsvParser.py @@ -0,0 +1,12 @@ +import csv + +from sources.file_types.AbstractParser import AbstractParser + + +class CsvParser(AbstractParser): + + def parse(self, source): + with open(source, 'r') as item: + reader = csv.DictReader(item) + for line in reader: + yield line \ No newline at end of file diff --git a/sources/source_types/AbstractSource.py b/sources/source_types/AbstractSource.py new file mode 100644 index 0000000..3208b7f --- /dev/null +++ b/sources/source_types/AbstractSource.py @@ -0,0 +1,37 @@ +from abc import ABC, abstractmethod + +from pydantic import Extra, BaseModel + +from sources.file_types.AbstractParser import AbstractParser + + +class AbstractSource(BaseModel, ABC): + file_hash: str = None + source_name: str = None + file_parser: AbstractParser = None + + class Config: + extra = Extra.allow + + def __init__(self, source_name, **kwargs): + super().__init__(**kwargs) + if not source_name: + raise Exception('You have to add "source_name" to your config!') + self.source_name = source_name + + @abstractmethod + def check_is_update_needed(self): + pass + + @abstractmethod + def parse_data(self): + pass + + def is_parser_needed(self): + return False + + def active_file_hash(self): + return self.file_hash + + def after_save(self): + pass \ No newline at end of file diff --git a/sources/source_types/FtpSource.py b/sources/source_types/FtpSource.py new file mode 100644 index 0000000..727c18f --- /dev/null +++ b/sources/source_types/FtpSource.py @@ -0,0 +1,58 @@ +import os + +from components.FileComponent import FileComponent +from components.FtpDownloader import FtpDownloader +from sources.source_types.AbstractSource import AbstractSource + + +class FtpSource(AbstractSource): + + # config + ftp_url: str + ftp_login: str + ftp_password: str + file_regex: str + local_files_path: str + + # additional params + source_files: list = [] + files_for_update: list = [] + local_files: list = [] + + def __init__(self, source_name, **kwargs): + super().__init__(source_name, **kwargs) + self.fileComponent = FileComponent(local_files_path=self.local_files_path) + self.local_files = self.fileComponent.get_parsed_files(source_name) + self.ftpClient = FtpDownloader(url=self.ftp_url, login=self.ftp_login, password=self.ftp_password) + self.source_files = self.ftpClient.get_files_with_checksum('/', self.file_regex) + self.__files_for_update() + + def __files_for_update(self): + local_hashes = [f.get('hash') for f in self.local_files] + self.files_for_update = [f for f in self.source_files if f.get('hash') not in local_hashes] + return self.files_for_update + + def is_parser_needed(self): + return True + + def download_file(self, file): + if not os.path.exists(self.local_files_path): + os.makedirs(self.local_files_path) + self.ftpClient.download_file(file, self.local_files_path + os.path.basename(file)) + + def check_is_update_needed(self): + return len(self.files_for_update) > 0 + + def parse_data(self): + for f in self.files_for_update: + self.download_file(f.get('file')) + self.file_hash = f.get('hash') + file_path = self.local_files_path + f.get('file') + data = self.file_parser.parse(file_path) + for line in data: + yield line + + def after_save(self): + for f in self.files_for_update: + self.fileComponent.add_parsed_file(self.source_name, f.get('file'), f.get('hash'), f.get('updated_at')) + self.fileComponent.remove_file(self.local_files_path + f.get('file')) \ No newline at end of file diff --git a/update.py b/update.py new file mode 100644 index 0000000..145e99d --- /dev/null +++ b/update.py @@ -0,0 +1,49 @@ +import argparse +from configs.config import config +from sources.BaseSource import BaseSource + + +class Updater: + def __init__(self): + parser = argparse.ArgumentParser(description="Source parser.") + parser.add_argument("--config", help='Update config action', action='store_true') + parser.add_argument("--source", type=str, help='List of sources: ' + ', '.join(config.get_sources_list())) + + self.args = parser.parse_args() + + def is_source_exists(self, source): + try: + config.get_source_by_name(source) + except ValueError: + return False + + return True + + def parse_source(self, source): + source_config = config.get_source_by_name(source) + source_model = BaseSource(**source_config) + + if not source_model.check_is_update_needed(): + print('Nothing to update') + return + else: + updated = source_model.update_data() + print('Rows added: ' + str(updated)) + + def update_config(self): + config.update_sources() + + def do_action(self): + if self.args.config: + self.update_config() + return + + if not self.is_source_exists(self.args.source): + print("This source not exists, list of sources: " + ', '.join(config.get_sources_list())) + return + + self.parse_source(self.args.source) + + +updater = Updater() +updater.do_action() From aaf98fc65eac8ea30cdbc58c2d13cd2823d07fd6 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Mon, 30 Jan 2023 10:44:20 +0300 Subject: [PATCH 02/35] add gitingnore --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2eea525 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.env \ No newline at end of file From e50cc762cf9259407c233fd7346d246027e945a4 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Mon, 30 Jan 2023 10:45:26 +0300 Subject: [PATCH 03/35] add build settings --- .env.sample | 5 +++++ build/build.xml | 41 +++++++++++++++++++++++++++++++++++++++++ configs/config.py | 24 ++++++------------------ configs/configs.py | 12 ++++++++++++ sources/BaseSource.py | 4 +++- 5 files changed, 67 insertions(+), 19 deletions(-) create mode 100644 .env.sample create mode 100644 build/build.xml create mode 100644 configs/configs.py diff --git a/.env.sample b/.env.sample new file mode 100644 index 0000000..ecf5645 --- /dev/null +++ b/.env.sample @@ -0,0 +1,5 @@ +DB_NAME=@lcDbName@ +DB_HOST=@lcDbHost@ + +API_URL=@lcApiUrl@ +API_KEY=@lcApiKey@ diff --git a/build/build.xml b/build/build.xml new file mode 100644 index 0000000..852f115 --- /dev/null +++ b/build/build.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/configs/config.py b/configs/config.py index 1b09ae5..af7f991 100644 --- a/configs/config.py +++ b/configs/config.py @@ -4,24 +4,19 @@ import requests from dynaconf import Dynaconf, Validator from mongoengine import connect +from configs.configs import CONFIGS + class AppConfig: def __init__(self): self.config = Dynaconf(settings_files=[ "/app/configs/main.json", - "/app/configs/application_credentials.json", - "/app/configs/db.json", "/app/configs/sources.json" ]) - self.config.validators.register( - Validator('db', 'db.host', 'db.database', must_exist=True), - ) - self.config.validators.validate() - creds = self.get_db_config() connect( - db=creds.database, - host="mongodb://{host}:27017/{database}".format(host=creds.host, database=creds.database) + db=CONFIGS['db']['name'], + host="mongodb://{host}:27017/{database}".format(host=CONFIGS['db']['host'], database=CONFIGS['db']['name']) ) def get_bulk_insert_limit(self): @@ -30,12 +25,6 @@ class AppConfig: else: return self.config.bulk_limit - def get_db_config(self): - return self.config.db - - def get_main_app_creds(self): - return self.config.application_credentials - def get_source_by_name(self, name: str): if name not in self.config.sources: raise ValueError(f'"{name}" source not exists!') @@ -45,12 +34,11 @@ class AppConfig: return self.config.sources.keys() def update_sources(self): - creds = self.get_main_app_creds() headers = { 'Content-type': 'application/json', - 'Authorization': 'Bearer {key}'.format(key=creds.api_key) + 'Authorization': 'Bearer {key}'.format(key=CONFIGS['application_credentials']['api_key']) } - r = requests.get(creds.api_url + "sources", headers=headers) + r = requests.get(CONFIGS['application_credentials']['api_url'] + "sources", headers=headers) if r.status_code != 200: raise Exception('Bad app response') diff --git a/configs/configs.py b/configs/configs.py new file mode 100644 index 0000000..7983a96 --- /dev/null +++ b/configs/configs.py @@ -0,0 +1,12 @@ +import os + +CONFIGS = { + 'db' : { + 'name' : os.environ['DB_NAME'], + 'host' : os.environ['DB_HOST'] + }, + 'application_credentials' : { + 'api_url' : os.environ['API_URL'], + 'api_key' : os.environ['API_KEY'] + } +} \ No newline at end of file diff --git a/sources/BaseSource.py b/sources/BaseSource.py index aac4d9d..a3c76b0 100644 --- a/sources/BaseSource.py +++ b/sources/BaseSource.py @@ -7,10 +7,12 @@ from pydantic import BaseModel, Extra from components.SenderComponent import SenderComponent from configs.config import config +from configs.configs import CONFIGS from models.MetaData import MetaData from sources.source_types.AbstractSource import AbstractSource + class BaseSource(BaseModel): source_name: str source: dict @@ -24,7 +26,7 @@ class BaseSource(BaseModel): def __init__(self, **kwargs): super().__init__(**kwargs) - self.senderComponent = SenderComponent(**config.get_main_app_creds()) + self.senderComponent = SenderComponent(api_url=CONFIGS['application_credentials']['api_url'], api_key=CONFIGS['application_credentials']['api_key']) def get_source_object(config): class_name = config.type.capitalize() + "Source" From 02ee18495bfce98151e2a8704e642a709aa8a7e4 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Mon, 30 Jan 2023 10:47:29 +0300 Subject: [PATCH 04/35] README.md update --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 028b29c..8ccdd54 100644 --- a/README.md +++ b/README.md @@ -72,3 +72,8 @@ Each source parser starts by crontab by command To see list of source types use command `python update.py -h` + + +## Run Updates + +Copy .env.sample to .env and update settings \ No newline at end of file From b60811668d3fbf808eb406a0827b0dee86d3d9f1 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Wed, 1 Feb 2023 14:24:56 +0300 Subject: [PATCH 05/35] update env file --- .env.sample | 8 ++++---- README.md | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.env.sample b/.env.sample index ecf5645..986cec6 100644 --- a/.env.sample +++ b/.env.sample @@ -1,5 +1,5 @@ -DB_NAME=@lcDbName@ -DB_HOST=@lcDbHost@ +DB_NAME="@lcDbName@" +DB_HOST="@lcDbHost@" -API_URL=@lcApiUrl@ -API_KEY=@lcApiKey@ +API_URL="@lcApiUrl@" +API_KEY="@lcApiKey@" diff --git a/README.md b/README.md index 8ccdd54..2aaaf48 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,7 @@ Added two components for working with CSV and FTP. • app ◦ components ◦ configs - ▪ application_credentials.json – keys and url for connection to our main app - ▪ db.json – creds for service db + ▪ configs.py – keys and url for connection to our main app and creds for service db ▪ main.json – main config ▪ sources.json – list of sources with components that they use ◦ models From 448367beee3fb6fa0bb148eedd88be157f33a9db Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Wed, 1 Feb 2023 15:29:16 +0300 Subject: [PATCH 06/35] update Dockerfile --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 6339c6c..c4ae0c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,3 +9,5 @@ RUN pip install pymongo \ mongoengine \ dict_hash \ requests + +COPY ./* /app/ From 01ec83ba91ae418d7222c33e2c64b46eb5f3d326 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Thu, 2 Feb 2023 12:34:32 +0300 Subject: [PATCH 07/35] remove copy target --- build/build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/build.xml b/build/build.xml index 852f115..96cb62d 100644 --- a/build/build.xml +++ b/build/build.xml @@ -3,7 +3,7 @@ From 217d63a3c0f9c7e9a52f3113eaa5f5eb823386ff Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Thu, 2 Feb 2023 12:36:13 +0300 Subject: [PATCH 08/35] update ant target --- build/build.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build/build.xml b/build/build.xml index 96cb62d..068ffbe 100644 --- a/build/build.xml +++ b/build/build.xml @@ -10,11 +10,11 @@ - - + + - - + + From e860892a5a5a69f00192dbf6cc63194147f2dd8a Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Fri, 3 Feb 2023 11:49:54 +0300 Subject: [PATCH 09/35] update copy --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c4ae0c2..d0a1fbb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,4 +10,4 @@ RUN pip install pymongo \ dict_hash \ requests -COPY ./* /app/ +COPY ./ /app/ From 8ef95e50f990d49d88e10100dc31fd46314d2736 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Fri, 3 Feb 2023 12:40:47 +0300 Subject: [PATCH 10/35] no message --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d0a1fbb..4cb171b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,4 +10,4 @@ RUN pip install pymongo \ dict_hash \ requests -COPY ./ /app/ +COPY ./ ./ From 93705b0a1d9a2ed9a19bab1c1cff39bbc13dee6e Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Fri, 3 Feb 2023 12:46:28 +0300 Subject: [PATCH 11/35] update configs --- configs/sources.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configs/sources.json b/configs/sources.json index e69de29..7a73a41 100644 --- a/configs/sources.json +++ b/configs/sources.json @@ -0,0 +1,2 @@ +{ +} \ No newline at end of file From 38a2df05a99eb1e90068b3910076a8432f30c2b7 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Mon, 6 Feb 2023 13:59:44 +0300 Subject: [PATCH 12/35] update config --- update.py | 1 - 1 file changed, 1 deletion(-) diff --git a/update.py b/update.py index 145e99d..8717659 100644 --- a/update.py +++ b/update.py @@ -7,7 +7,6 @@ class Updater: def __init__(self): parser = argparse.ArgumentParser(description="Source parser.") parser.add_argument("--config", help='Update config action', action='store_true') - parser.add_argument("--source", type=str, help='List of sources: ' + ', '.join(config.get_sources_list())) self.args = parser.parse_args() From 8ea0424242e362e426974686cf7ad01fcabe4761 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 7 Feb 2023 10:41:34 +0300 Subject: [PATCH 13/35] run scripts --- Dockerfile | 7 +++++++ update.py | 1 + 2 files changed, 8 insertions(+) diff --git a/Dockerfile b/Dockerfile index 4cb171b..2c6d548 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,3 +11,10 @@ RUN pip install pymongo \ requests COPY ./ ./ + +RUN chmod 777 /app/configs/sources.json + +RUN python update.py --config + +RUN python update.py --source test + diff --git a/update.py b/update.py index 8717659..eecdbdb 100644 --- a/update.py +++ b/update.py @@ -7,6 +7,7 @@ class Updater: def __init__(self): parser = argparse.ArgumentParser(description="Source parser.") parser.add_argument("--config", help='Update config action', action='store_true') + parser.add_argument("--source", type=str) self.args = parser.parse_args() From 8f926f714b3033dd24cb25ef0f0a8e54d083504d Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 7 Feb 2023 11:57:49 +0300 Subject: [PATCH 14/35] refresh configs --- Dockerfile | 4 ---- configs/config.py | 17 +++++++++++------ update.py | 5 ++++- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2c6d548..23d4cf5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,3 @@ COPY ./ ./ RUN chmod 777 /app/configs/sources.json -RUN python update.py --config - -RUN python update.py --source test - diff --git a/configs/config.py b/configs/config.py index af7f991..f0cfbf2 100644 --- a/configs/config.py +++ b/configs/config.py @@ -9,16 +9,18 @@ from configs.configs import CONFIGS class AppConfig: def __init__(self): - self.config = Dynaconf(settings_files=[ - "/app/configs/main.json", - "/app/configs/sources.json" - ]) - + self.refresh() connect( db=CONFIGS['db']['name'], host="mongodb://{host}:27017/{database}".format(host=CONFIGS['db']['host'], database=CONFIGS['db']['name']) ) - + + def refresh(self): + self.config = Dynaconf(settings_files=[ + "/app/configs/main.json", + "/app/configs/sources.json" + ]) + def get_bulk_insert_limit(self): if not self.config.bulk_limit: return 1 @@ -40,6 +42,7 @@ class AppConfig: } r = requests.get(CONFIGS['application_credentials']['api_url'] + "sources", headers=headers) if r.status_code != 200: + raise Exception('Bad app response') new_config = {"sources": {}} @@ -48,5 +51,7 @@ class AppConfig: with open("./configs/sources.json", "w") as outfile: outfile.write(json.dumps(new_config)) + + self.refresh() config = AppConfig() \ No newline at end of file diff --git a/update.py b/update.py index eecdbdb..913a1d0 100644 --- a/update.py +++ b/update.py @@ -34,14 +34,17 @@ class Updater: config.update_sources() def do_action(self): + print("Start Updater"); if self.args.config: + print("Start Configure") self.update_config() return if not self.is_source_exists(self.args.source): print("This source not exists, list of sources: " + ', '.join(config.get_sources_list())) return - + + print("Start source update") self.parse_source(self.args.source) From 14c011b921ab46bf4c9f601916c7ec79d72fa529 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 7 Feb 2023 11:58:52 +0300 Subject: [PATCH 15/35] update whitespace --- update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update.py b/update.py index 913a1d0..3edb210 100644 --- a/update.py +++ b/update.py @@ -44,7 +44,7 @@ class Updater: print("This source not exists, list of sources: " + ', '.join(config.get_sources_list())) return - print("Start source update") + print("Start source update") self.parse_source(self.args.source) From 2615ce4b1d5935d31ee1fe0cb14abbce0c7324a5 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 7 Feb 2023 12:11:39 +0300 Subject: [PATCH 16/35] run with all params --- configs/config.py | 8 ++++---- update.py | 5 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/configs/config.py b/configs/config.py index f0cfbf2..62688b8 100644 --- a/configs/config.py +++ b/configs/config.py @@ -9,14 +9,14 @@ from configs.configs import CONFIGS class AppConfig: def __init__(self): - self.refresh() + self.refresh() connect( db=CONFIGS['db']['name'], host="mongodb://{host}:27017/{database}".format(host=CONFIGS['db']['host'], database=CONFIGS['db']['name']) ) - def refresh(self): - self.config = Dynaconf(settings_files=[ + def refresh(self): + self.config = Dynaconf(settings_files=[ "/app/configs/main.json", "/app/configs/sources.json" ]) @@ -52,6 +52,6 @@ class AppConfig: with open("./configs/sources.json", "w") as outfile: outfile.write(json.dumps(new_config)) - self.refresh() + self.refresh() config = AppConfig() \ No newline at end of file diff --git a/update.py b/update.py index 3edb210..5c68fb3 100644 --- a/update.py +++ b/update.py @@ -34,11 +34,10 @@ class Updater: config.update_sources() def do_action(self): - print("Start Updater"); + print("Start Updater"); if self.args.config: - print("Start Configure") + print("Start Configure") self.update_config() - return if not self.is_source_exists(self.args.source): print("This source not exists, list of sources: " + ', '.join(config.get_sources_list())) From 4ac5becb02e5509a655c791807e6d8a9a9a20561 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 7 Feb 2023 12:19:06 +0300 Subject: [PATCH 17/35] update build env --- .env.sample | 2 ++ build/build.xml | 2 ++ configs/config.py | 2 ++ configs/configs.py | 4 +++- 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.env.sample b/.env.sample index 986cec6..5adb97b 100644 --- a/.env.sample +++ b/.env.sample @@ -1,5 +1,7 @@ DB_NAME="@lcDbName@" DB_HOST="@lcDbHost@" +DB_USERNAME="@lcDbUsername@" +DB_PASSWORD="@lcDbPassword@" API_URL="@lcApiUrl@" API_KEY="@lcApiKey@" diff --git a/build/build.xml b/build/build.xml index 068ffbe..e2d3d72 100644 --- a/build/build.xml +++ b/build/build.xml @@ -12,6 +12,8 @@ description="Replace placeholders in config files" > + + diff --git a/configs/config.py b/configs/config.py index 62688b8..6417a05 100644 --- a/configs/config.py +++ b/configs/config.py @@ -12,6 +12,8 @@ class AppConfig: self.refresh() connect( db=CONFIGS['db']['name'], + username=CONFIGS['db']['username'], + password=CONFIGS['db']['password'] host="mongodb://{host}:27017/{database}".format(host=CONFIGS['db']['host'], database=CONFIGS['db']['name']) ) diff --git a/configs/configs.py b/configs/configs.py index 7983a96..3765b12 100644 --- a/configs/configs.py +++ b/configs/configs.py @@ -3,7 +3,9 @@ import os CONFIGS = { 'db' : { 'name' : os.environ['DB_NAME'], - 'host' : os.environ['DB_HOST'] + 'host' : os.environ['DB_HOST'], + 'username' : os.environ['DB_USERNAME'], + 'password' : os.environ['DB_PASSWORD'] }, 'application_credentials' : { 'api_url' : os.environ['API_URL'], From 26f8e2dcbb9dd86859699ca4e0aa8afdf86d13cf Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 7 Feb 2023 12:22:12 +0300 Subject: [PATCH 18/35] update configs --- configs/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/config.py b/configs/config.py index 6417a05..c023820 100644 --- a/configs/config.py +++ b/configs/config.py @@ -12,8 +12,8 @@ class AppConfig: self.refresh() connect( db=CONFIGS['db']['name'], - username=CONFIGS['db']['username'], - password=CONFIGS['db']['password'] + username=CONFIGS['db']['username'], + password=CONFIGS['db']['password'] host="mongodb://{host}:27017/{database}".format(host=CONFIGS['db']['host'], database=CONFIGS['db']['name']) ) From 9042b4e776b2343b52f71356ea9d7104cc1dee86 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 7 Feb 2023 14:26:43 +0300 Subject: [PATCH 19/35] update configs --- configs/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configs/config.py b/configs/config.py index c023820..e7a3e35 100644 --- a/configs/config.py +++ b/configs/config.py @@ -13,7 +13,8 @@ class AppConfig: connect( db=CONFIGS['db']['name'], username=CONFIGS['db']['username'], - password=CONFIGS['db']['password'] + password=CONFIGS['db']['password'], + authentication_source='admin', host="mongodb://{host}:27017/{database}".format(host=CONFIGS['db']['host'], database=CONFIGS['db']['name']) ) From f167b1e5a2a008da2b2cae7359c2bd234dc4145e Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 7 Feb 2023 14:27:37 +0300 Subject: [PATCH 20/35] bug fix --- configs/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/config.py b/configs/config.py index e7a3e35..73314c3 100644 --- a/configs/config.py +++ b/configs/config.py @@ -14,7 +14,7 @@ class AppConfig: db=CONFIGS['db']['name'], username=CONFIGS['db']['username'], password=CONFIGS['db']['password'], - authentication_source='admin', + authentication_source='admin', host="mongodb://{host}:27017/{database}".format(host=CONFIGS['db']['host'], database=CONFIGS['db']['name']) ) From ecb3166dfed49d79f813b45ea948450d046372d9 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 10:04:51 +0300 Subject: [PATCH 21/35] update run --- Dockerfile | 1 + update.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 23d4cf5..672d2e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,4 +13,5 @@ RUN pip install pymongo \ COPY ./ ./ RUN chmod 777 /app/configs/sources.json +RUN python /app/update.py diff --git a/update.py b/update.py index 5c68fb3..7f11639 100644 --- a/update.py +++ b/update.py @@ -35,16 +35,17 @@ class Updater: def do_action(self): print("Start Updater"); - if self.args.config: - print("Start Configure") - self.update_config() + print("Start Configure") + self.update_config() - if not self.is_source_exists(self.args.source): - print("This source not exists, list of sources: " + ', '.join(config.get_sources_list())) + if self.is_source_exists(self.args.source): + print("Start source update" + self.args.source) + self.parse_source(self.args.source) return - print("Start source update") - self.parse_source(self.args.source) + for source in self.args.source: + print("Start source update" + source) + self.parse_source(source) updater = Updater() From 12011220e5d45524a34610adc83a6793d65eded7 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 10:06:09 +0300 Subject: [PATCH 22/35] remove whilespace --- update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update.py b/update.py index 7f11639..45bf343 100644 --- a/update.py +++ b/update.py @@ -43,7 +43,7 @@ class Updater: self.parse_source(self.args.source) return - for source in self.args.source: + for source in self.args.source: print("Start source update" + source) self.parse_source(source) From 8cef5a5e523d30660263df58e721b8f4b70a9a5b Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 10:11:27 +0300 Subject: [PATCH 23/35] small fixes --- update.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/update.py b/update.py index 45bf343..ecf28dc 100644 --- a/update.py +++ b/update.py @@ -38,12 +38,12 @@ class Updater: print("Start Configure") self.update_config() - if self.is_source_exists(self.args.source): + if hasattr(self.args, 'source') && self.is_source_exists(self.args.source): print("Start source update" + self.args.source) self.parse_source(self.args.source) return - for source in self.args.source: + for source in config.get_sources_list(): print("Start source update" + source) self.parse_source(source) From 4f115e3337e74c176f89312efd5a38b51337cbee Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 10:19:17 +0300 Subject: [PATCH 24/35] show sources --- update.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/update.py b/update.py index ecf28dc..ebd2fa5 100644 --- a/update.py +++ b/update.py @@ -37,8 +37,9 @@ class Updater: print("Start Updater"); print("Start Configure") self.update_config() - - if hasattr(self.args, 'source') && self.is_source_exists(self.args.source): + config.get_sources_list() + print(dir(config)) + if hasattr(self.args, 'source') and self.is_source_exists(self.args.source): print("Start source update" + self.args.source) self.parse_source(self.args.source) return From c3b74ced85267e8695d8a5de4f87287b1d305677 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 10:26:44 +0300 Subject: [PATCH 25/35] add debug info --- configs/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/config.py b/configs/config.py index 73314c3..afc0183 100644 --- a/configs/config.py +++ b/configs/config.py @@ -51,7 +51,7 @@ class AppConfig: new_config = {"sources": {}} for source in r.json(): new_config["sources"][source["source_name"]] = source - + print(dir(new_config)) with open("./configs/sources.json", "w") as outfile: outfile.write(json.dumps(new_config)) From 1abfc53ffd38970f08f884f53ed7c46e959b35e0 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 10:32:23 +0300 Subject: [PATCH 26/35] add debug --- update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update.py b/update.py index ebd2fa5..a069d76 100644 --- a/update.py +++ b/update.py @@ -37,8 +37,8 @@ class Updater: print("Start Updater"); print("Start Configure") self.update_config() - config.get_sources_list() print(dir(config)) + return if hasattr(self.args, 'source') and self.is_source_exists(self.args.source): print("Start source update" + self.args.source) self.parse_source(self.args.source) From 637fafff5e1419fce360e40eab1f90c3bb90a870 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 10:33:20 +0300 Subject: [PATCH 27/35] remove whitespace --- update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update.py b/update.py index a069d76..4a09cc1 100644 --- a/update.py +++ b/update.py @@ -38,7 +38,7 @@ class Updater: print("Start Configure") self.update_config() print(dir(config)) - return + return if hasattr(self.args, 'source') and self.is_source_exists(self.args.source): print("Start source update" + self.args.source) self.parse_source(self.args.source) From 7a6d84dc20d68f70bc5816a58d6eb8e64edc21f7 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 13:29:26 +0300 Subject: [PATCH 28/35] updates --- Dockerfile | 4 +--- configs/config.py | 2 +- update.py | 2 -- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 672d2e6..33b7c32 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,4 @@ RUN pip install pymongo \ COPY ./ ./ -RUN chmod 777 /app/configs/sources.json -RUN python /app/update.py - +CMD ["python", "update.py"] \ No newline at end of file diff --git a/configs/config.py b/configs/config.py index afc0183..4e31ecc 100644 --- a/configs/config.py +++ b/configs/config.py @@ -51,7 +51,7 @@ class AppConfig: new_config = {"sources": {}} for source in r.json(): new_config["sources"][source["source_name"]] = source - print(dir(new_config)) + with open("./configs/sources.json", "w") as outfile: outfile.write(json.dumps(new_config)) diff --git a/update.py b/update.py index 4a09cc1..c8cf52f 100644 --- a/update.py +++ b/update.py @@ -37,8 +37,6 @@ class Updater: print("Start Updater"); print("Start Configure") self.update_config() - print(dir(config)) - return if hasattr(self.args, 'source') and self.is_source_exists(self.args.source): print("Start source update" + self.args.source) self.parse_source(self.args.source) From 327e3e9bbcc439c45218dcbbb66251a978c6287a Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 13:40:34 +0300 Subject: [PATCH 29/35] add exceprions --- update.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/update.py b/update.py index c8cf52f..f22c4bf 100644 --- a/update.py +++ b/update.py @@ -43,8 +43,11 @@ class Updater: return for source in config.get_sources_list(): - print("Start source update" + source) - self.parse_source(source) + try: + print("Start source update" + source) + self.parse_source(source) + except: + print(source + " has error") updater = Updater() From 78b86ad9659d61919db3db2a543c6dd754ce8337 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 14:39:23 +0300 Subject: [PATCH 30/35] update entrypoint --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 33b7c32..165251a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,4 +12,6 @@ RUN pip install pymongo \ COPY ./ ./ -CMD ["python", "update.py"] \ No newline at end of file +ENTRYPOINT ["python3"] + +CMD ["update.py"] \ No newline at end of file From 3d8d362b08c05878fdfd79920ffef8f57060ed8d Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 14:52:28 +0300 Subject: [PATCH 31/35] remove cmd --- Dockerfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 165251a..2500655 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,8 +10,4 @@ RUN pip install pymongo \ dict_hash \ requests -COPY ./ ./ - -ENTRYPOINT ["python3"] - -CMD ["update.py"] \ No newline at end of file +COPY ./ ./ \ No newline at end of file From 3b81d997a9fd3621f1a53d1b0707ba22a14d8c72 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 15:34:36 +0300 Subject: [PATCH 32/35] show env --- configs/configs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configs/configs.py b/configs/configs.py index 3765b12..23b1e65 100644 --- a/configs/configs.py +++ b/configs/configs.py @@ -1,5 +1,8 @@ import os +for name, value in os.environ.items(): + print("{0}: {1}".format(name, value)) + CONFIGS = { 'db' : { 'name' : os.environ['DB_NAME'], From d7eccd92afb454c486608f2a8b0090a294523b48 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 19:17:05 +0300 Subject: [PATCH 33/35] update configs --- configs/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configs/config.py b/configs/config.py index 4e31ecc..51be7e1 100644 --- a/configs/config.py +++ b/configs/config.py @@ -15,7 +15,8 @@ class AppConfig: username=CONFIGS['db']['username'], password=CONFIGS['db']['password'], authentication_source='admin', - host="mongodb://{host}:27017/{database}".format(host=CONFIGS['db']['host'], database=CONFIGS['db']['name']) + port=27017, + host=CONFIGS['db']['host']) ) def refresh(self): From 91daad730fa3f314f635a22a186f57b6f6a2a1f2 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 19:20:42 +0300 Subject: [PATCH 34/35] bug fix --- configs/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/config.py b/configs/config.py index 51be7e1..9ec72bc 100644 --- a/configs/config.py +++ b/configs/config.py @@ -16,7 +16,7 @@ class AppConfig: password=CONFIGS['db']['password'], authentication_source='admin', port=27017, - host=CONFIGS['db']['host']) + host=CONFIGS['db']['host'] ) def refresh(self): From 47dec26d88c48e15de6addc640d9069cc146a32e Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Wed, 22 Feb 2023 09:19:23 +0300 Subject: [PATCH 35/35] remove debug info --- configs/configs.py | 3 --- update.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/configs/configs.py b/configs/configs.py index 23b1e65..3765b12 100644 --- a/configs/configs.py +++ b/configs/configs.py @@ -1,8 +1,5 @@ import os -for name, value in os.environ.items(): - print("{0}: {1}".format(name, value)) - CONFIGS = { 'db' : { 'name' : os.environ['DB_NAME'], diff --git a/update.py b/update.py index f22c4bf..bbe8b44 100644 --- a/update.py +++ b/update.py @@ -38,7 +38,7 @@ class Updater: print("Start Configure") self.update_config() if hasattr(self.args, 'source') and self.is_source_exists(self.args.source): - print("Start source update" + self.args.source) + print("Start single source" + self.args.source) self.parse_source(self.args.source) return