From ecb3166dfed49d79f813b45ea948450d046372d9 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 10:04:51 +0300 Subject: [PATCH 01/15] update run --- Dockerfile | 1 + update.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 23d4cf5..672d2e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,4 +13,5 @@ RUN pip install pymongo \ COPY ./ ./ RUN chmod 777 /app/configs/sources.json +RUN python /app/update.py diff --git a/update.py b/update.py index 5c68fb3..7f11639 100644 --- a/update.py +++ b/update.py @@ -35,16 +35,17 @@ class Updater: def do_action(self): print("Start Updater"); - if self.args.config: - print("Start Configure") - self.update_config() + print("Start Configure") + self.update_config() - if not self.is_source_exists(self.args.source): - print("This source not exists, list of sources: " + ', '.join(config.get_sources_list())) + if self.is_source_exists(self.args.source): + print("Start source update" + self.args.source) + self.parse_source(self.args.source) return - print("Start source update") - self.parse_source(self.args.source) + for source in self.args.source: + print("Start source update" + source) + self.parse_source(source) updater = Updater() From 12011220e5d45524a34610adc83a6793d65eded7 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 10:06:09 +0300 Subject: [PATCH 02/15] remove whilespace --- update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update.py b/update.py index 7f11639..45bf343 100644 --- a/update.py +++ b/update.py @@ -43,7 +43,7 @@ class Updater: self.parse_source(self.args.source) return - for source in self.args.source: + for source in self.args.source: print("Start source update" + source) self.parse_source(source) From 8cef5a5e523d30660263df58e721b8f4b70a9a5b Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 10:11:27 +0300 Subject: [PATCH 03/15] small fixes --- update.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/update.py b/update.py index 45bf343..ecf28dc 100644 --- a/update.py +++ b/update.py @@ -38,12 +38,12 @@ class Updater: print("Start Configure") self.update_config() - if self.is_source_exists(self.args.source): + if hasattr(self.args, 'source') && self.is_source_exists(self.args.source): print("Start source update" + self.args.source) self.parse_source(self.args.source) return - for source in self.args.source: + for source in config.get_sources_list(): print("Start source update" + source) self.parse_source(source) From 4f115e3337e74c176f89312efd5a38b51337cbee Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 10:19:17 +0300 Subject: [PATCH 04/15] show sources --- update.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/update.py b/update.py index ecf28dc..ebd2fa5 100644 --- a/update.py +++ b/update.py @@ -37,8 +37,9 @@ class Updater: print("Start Updater"); print("Start Configure") self.update_config() - - if hasattr(self.args, 'source') && self.is_source_exists(self.args.source): + config.get_sources_list() + print(dir(config)) + if hasattr(self.args, 'source') and self.is_source_exists(self.args.source): print("Start source update" + self.args.source) self.parse_source(self.args.source) return From c3b74ced85267e8695d8a5de4f87287b1d305677 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 10:26:44 +0300 Subject: [PATCH 05/15] add debug info --- configs/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/config.py b/configs/config.py index 73314c3..afc0183 100644 --- a/configs/config.py +++ b/configs/config.py @@ -51,7 +51,7 @@ class AppConfig: new_config = {"sources": {}} for source in r.json(): new_config["sources"][source["source_name"]] = source - + print(dir(new_config)) with open("./configs/sources.json", "w") as outfile: outfile.write(json.dumps(new_config)) From 1abfc53ffd38970f08f884f53ed7c46e959b35e0 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 10:32:23 +0300 Subject: [PATCH 06/15] add debug --- update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update.py b/update.py index ebd2fa5..a069d76 100644 --- a/update.py +++ b/update.py @@ -37,8 +37,8 @@ class Updater: print("Start Updater"); print("Start Configure") self.update_config() - config.get_sources_list() print(dir(config)) + return if hasattr(self.args, 'source') and self.is_source_exists(self.args.source): print("Start source update" + self.args.source) self.parse_source(self.args.source) From 637fafff5e1419fce360e40eab1f90c3bb90a870 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 10:33:20 +0300 Subject: [PATCH 07/15] remove whitespace --- update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update.py b/update.py index a069d76..4a09cc1 100644 --- a/update.py +++ b/update.py @@ -38,7 +38,7 @@ class Updater: print("Start Configure") self.update_config() print(dir(config)) - return + return if hasattr(self.args, 'source') and self.is_source_exists(self.args.source): print("Start source update" + self.args.source) self.parse_source(self.args.source) From 7a6d84dc20d68f70bc5816a58d6eb8e64edc21f7 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 13:29:26 +0300 Subject: [PATCH 08/15] updates --- Dockerfile | 4 +--- configs/config.py | 2 +- update.py | 2 -- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 672d2e6..33b7c32 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,4 @@ RUN pip install pymongo \ COPY ./ ./ -RUN chmod 777 /app/configs/sources.json -RUN python /app/update.py - +CMD ["python", "update.py"] \ No newline at end of file diff --git a/configs/config.py b/configs/config.py index afc0183..4e31ecc 100644 --- a/configs/config.py +++ b/configs/config.py @@ -51,7 +51,7 @@ class AppConfig: new_config = {"sources": {}} for source in r.json(): new_config["sources"][source["source_name"]] = source - print(dir(new_config)) + with open("./configs/sources.json", "w") as outfile: outfile.write(json.dumps(new_config)) diff --git a/update.py b/update.py index 4a09cc1..c8cf52f 100644 --- a/update.py +++ b/update.py @@ -37,8 +37,6 @@ class Updater: print("Start Updater"); print("Start Configure") self.update_config() - print(dir(config)) - return if hasattr(self.args, 'source') and self.is_source_exists(self.args.source): print("Start source update" + self.args.source) self.parse_source(self.args.source) From 327e3e9bbcc439c45218dcbbb66251a978c6287a Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 13:40:34 +0300 Subject: [PATCH 09/15] add exceprions --- update.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/update.py b/update.py index c8cf52f..f22c4bf 100644 --- a/update.py +++ b/update.py @@ -43,8 +43,11 @@ class Updater: return for source in config.get_sources_list(): - print("Start source update" + source) - self.parse_source(source) + try: + print("Start source update" + source) + self.parse_source(source) + except: + print(source + " has error") updater = Updater() From 78b86ad9659d61919db3db2a543c6dd754ce8337 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 14:39:23 +0300 Subject: [PATCH 10/15] update entrypoint --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 33b7c32..165251a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,4 +12,6 @@ RUN pip install pymongo \ COPY ./ ./ -CMD ["python", "update.py"] \ No newline at end of file +ENTRYPOINT ["python3"] + +CMD ["update.py"] \ No newline at end of file From 3d8d362b08c05878fdfd79920ffef8f57060ed8d Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 14:52:28 +0300 Subject: [PATCH 11/15] remove cmd --- Dockerfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 165251a..2500655 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,8 +10,4 @@ RUN pip install pymongo \ dict_hash \ requests -COPY ./ ./ - -ENTRYPOINT ["python3"] - -CMD ["update.py"] \ No newline at end of file +COPY ./ ./ \ No newline at end of file From 3b81d997a9fd3621f1a53d1b0707ba22a14d8c72 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 15:34:36 +0300 Subject: [PATCH 12/15] show env --- configs/configs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configs/configs.py b/configs/configs.py index 3765b12..23b1e65 100644 --- a/configs/configs.py +++ b/configs/configs.py @@ -1,5 +1,8 @@ import os +for name, value in os.environ.items(): + print("{0}: {1}".format(name, value)) + CONFIGS = { 'db' : { 'name' : os.environ['DB_NAME'], From d7eccd92afb454c486608f2a8b0090a294523b48 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 19:17:05 +0300 Subject: [PATCH 13/15] update configs --- configs/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configs/config.py b/configs/config.py index 4e31ecc..51be7e1 100644 --- a/configs/config.py +++ b/configs/config.py @@ -15,7 +15,8 @@ class AppConfig: username=CONFIGS['db']['username'], password=CONFIGS['db']['password'], authentication_source='admin', - host="mongodb://{host}:27017/{database}".format(host=CONFIGS['db']['host'], database=CONFIGS['db']['name']) + port=27017, + host=CONFIGS['db']['host']) ) def refresh(self): From 91daad730fa3f314f635a22a186f57b6f6a2a1f2 Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Tue, 21 Feb 2023 19:20:42 +0300 Subject: [PATCH 14/15] bug fix --- configs/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/config.py b/configs/config.py index 51be7e1..9ec72bc 100644 --- a/configs/config.py +++ b/configs/config.py @@ -16,7 +16,7 @@ class AppConfig: password=CONFIGS['db']['password'], authentication_source='admin', port=27017, - host=CONFIGS['db']['host']) + host=CONFIGS['db']['host'] ) def refresh(self): From 47dec26d88c48e15de6addc640d9069cc146a32e Mon Sep 17 00:00:00 2001 From: Egor Svitin Date: Wed, 22 Feb 2023 09:19:23 +0300 Subject: [PATCH 15/15] remove debug info --- configs/configs.py | 3 --- update.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/configs/configs.py b/configs/configs.py index 23b1e65..3765b12 100644 --- a/configs/configs.py +++ b/configs/configs.py @@ -1,8 +1,5 @@ import os -for name, value in os.environ.items(): - print("{0}: {1}".format(name, value)) - CONFIGS = { 'db' : { 'name' : os.environ['DB_NAME'], diff --git a/update.py b/update.py index f22c4bf..bbe8b44 100644 --- a/update.py +++ b/update.py @@ -38,7 +38,7 @@ class Updater: print("Start Configure") self.update_config() if hasattr(self.args, 'source') and self.is_source_exists(self.args.source): - print("Start source update" + self.args.source) + print("Start single source" + self.args.source) self.parse_source(self.args.source) return