diff --git a/src/crawler/by_source/eudml_crawler.py b/src/crawler/by_source/eudml_crawler.py index 466e163a4825b0f5ece2a38364aaa4fc9777c4a6..cb5355cba2eab5e497d52717f25a1b87ff7f2688 100644 --- a/src/crawler/by_source/eudml_crawler.py +++ b/src/crawler/by_source/eudml_crawler.py @@ -89,6 +89,7 @@ class EudmlCrawler(BaseCollectionCrawler): txt = f"EuDML declares {volumes_to_find} volumes for {self.collection_id}. We parsed {volume_count}" print(txt) if settings.CRAWLER_LOG_FILE: + os.makedirs(os.path.dirname(settings.CRAWLER_LOG_FILE), exist_ok=True) with open(settings.CRAWLER_LOG_FILE, "a") as f_: f_.write(txt + "\n") @@ -96,6 +97,7 @@ class EudmlCrawler(BaseCollectionCrawler): txt = f"EuDML declares {issues_to_find} issues for {self.collection_id}. We parsed {issue_count}" print(txt) if settings.CRAWLER_LOG_FILE: + os.makedirs(os.path.dirname(settings.CRAWLER_LOG_FILE), exist_ok=True) with open(settings.CRAWLER_LOG_FILE, "a") as f_: f_.write(txt + "\n") @@ -104,6 +106,7 @@ class EudmlCrawler(BaseCollectionCrawler): txt = f"EuDML declares {articles_to_find} articles for {self.collection_id}. We parsed {article_count}" print(txt) if settings.CRAWLER_LOG_FILE: + os.makedirs(os.path.dirname(settings.CRAWLER_LOG_FILE), exist_ok=True) with open(settings.CRAWLER_LOG_FILE, "a") as f_: f_.write(txt + "\n") diff --git a/src/crawler/settings.py b/src/crawler/settings.py index 478399a159790d6ff30ab299f2d09eca6b9060b2..323995fb8fe481c82da46c800a84b7327642ec39 100644 --- a/src/crawler/settings.py +++ b/src/crawler/settings.py @@ -1,7 +1,7 @@ from ptf.settings import * # noqa: F403 from ptf.site_register import SITE_REGISTER -CRAWLER_LOG_FILE = "" # Override in settings_local.py +CRAWLER_LOG_FILE = None # Override in settings_local.py TIME_ZONE = "UTC" USE_I18N = True