Skip to content

Commit 82f9f51

Browse files
kdp-cloudkdp-cloud
authored
Merge pull request #98 from elixir-europe/cleanup_and_fix_datafile_accessions
Cleanup and fix datafile accessions
2 parents 8ee3540 + b416aba commit 82f9f51

13 files changed

+3760
-212
lines changed

‎mars-cli/mars_cli.py

+13-149
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,23 @@
11
import click
2-
import logging
3-
import pathlib
4-
from configparser import ConfigParser
52
from datetime import datetime
63
from mars_lib.target_repo import TargetRepository
74
from mars_lib.models.isa_json import IsaJson
85
from mars_lib.submit import submission
96
from mars_lib.credential import CredentialManager
10-
from mars_lib.logging import print_and_log
7+
from mars_lib.logging import print_and_log, init_logging
118
from mars_lib.validation import validate, CustomValidationException
12-
from logging.handlers import RotatingFileHandler
139
import requests
1410
import sys
15-
import os
1611
import json
12+
from pathlib import Path
13+
import os
14+
from configparser import ConfigParser
1715

1816
# Load CLI configuration
1917
home_dir = (
20-
pathlib.Path(str(os.getenv("MARS_SETTINGS_DIR")))
18+
Path(str(os.getenv("MARS_SETTINGS_DIR")))
2119
if os.getenv("MARS_SETTINGS_DIR")
22-
else pathlib.Path.home()
20+
else Path.home()
2321
)
2422

2523
config_file = home_dir / ".mars" / "settings.ini"
@@ -28,149 +26,11 @@
2826
config = ConfigParser()
2927
config.read(config_file)
3028

31-
# Logging configuration
32-
log_level = config.get("logging", "log_level", fallback="ERROR")
33-
log_file = config.get("logging", "log_file", fallback=fallback_log_file)
34-
log_max_size = int(
35-
config.get("logging", "log_max_size", fallback="1024")
36-
) # in kilobytes. 1 MB by default.
37-
log_max_files = int(
38-
config.get("logging", "log_max_files", fallback="5")
39-
) # number of backup files. 5 by default.
40-
41-
handler = RotatingFileHandler(
42-
log_file, maxBytes=log_max_size * 1024, backupCount=log_max_files
43-
)
44-
handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s"))
45-
46-
logging.basicConfig(
47-
handlers=[handler],
48-
level=log_level,
49-
)
29+
# Load logging configuration
30+
init_logging(config, fallback_log_file)
5031

5132
# Read in all the URLs from the config file
52-
urls = {
53-
"DEV": {
54-
"ENA": {
55-
"SERVICE": config.get(
56-
"ena",
57-
"development-url",
58-
fallback="https://wwwdev.ebi.ac.uk/biosamples/samples",
59-
),
60-
"SUBMISSION": config.get(
61-
"ena",
62-
"development-submission-url",
63-
fallback="https://wwwdev.ebi.ac.uk/biosamples/samples/submit",
64-
),
65-
"DATA-SUBMISSION": config.get(
66-
"ena",
67-
"development-data-submission-url",
68-
fallback="webin2.ebi.ac.uk",
69-
),
70-
},
71-
"WEBIN": {
72-
"SERVICE": config.get(
73-
"webin",
74-
"development-url",
75-
fallback="https://wwwdev.ebi.ac.uk/ena/submit/webin/auth",
76-
),
77-
"TOKEN": config.get(
78-
"webin",
79-
"development-token-url",
80-
fallback="https://wwwdev.ebi.ac.uk/ena/submit/webin/auth/token",
81-
),
82-
},
83-
"METABOLIGHTS": {
84-
"SERVICE": config.get(
85-
"metabolights",
86-
"development-url",
87-
fallback="https://www-test.ebi.ac.uk/metabolights/mars/ws3/submissions/",
88-
),
89-
"SUBMISSION": config.get(
90-
"metabolights",
91-
"development-submission-url",
92-
fallback="https://www-test.ebi.ac.uk/metabolights/mars/ws3/submissions/",
93-
),
94-
"TOKEN": config.get(
95-
"metabolights",
96-
"development-token-url",
97-
fallback="https://www-test.ebi.ac.uk/metabolights/mars/ws3/auth/token",
98-
),
99-
},
100-
"BIOSAMPLES": {
101-
"SERVICE": config.get(
102-
"biosamples",
103-
"development-url",
104-
fallback="https://wwwdev.ebi.ac.uk/biosamples/samples/",
105-
),
106-
"SUBMISSION": config.get(
107-
"biosamples",
108-
"development-submission-url",
109-
fallback="https://wwwdev.ebi.ac.uk/biosamples/samples/",
110-
),
111-
},
112-
},
113-
"PROD": {
114-
"ENA": {
115-
"SERVICE": config.get(
116-
"ena",
117-
"production-url",
118-
fallback="https://www.ebi.ac.uk/ena/submit/webin-v2/",
119-
),
120-
"SUBMISSION": config.get(
121-
"ena",
122-
"production-submission-url",
123-
fallback="https://www.ebi.ac.uk/ena/submit/drop-box/submit/?auth=ENA",
124-
),
125-
"DATA-SUBMISSION": config.get(
126-
"ena",
127-
"development-data-submission-url",
128-
fallback="webin2.ebi.ac.uk",
129-
),
130-
},
131-
"WEBIN": {
132-
"SERVICE": config.get(
133-
"webin",
134-
"production-url",
135-
fallback="https://wwwdev.ebi.ac.uk/ena/dev/submit/webin/auth",
136-
),
137-
"TOKEN": config.get(
138-
"webin",
139-
"production-token-url",
140-
fallback="https://wwwdev.ebi.ac.uk/ena/dev/submit/webin/auth/token",
141-
),
142-
},
143-
"METABOLIGHTS": {
144-
"SERVICE": config.get(
145-
"metabolights",
146-
"production-url",
147-
fallback="https://www-test.ebi.ac.uk/metabolights/mars/ws3/submissions/",
148-
),
149-
"SUBMISSION": config.get(
150-
"metabolights",
151-
"production-submission-url",
152-
fallback="https://www-test.ebi.ac.uk/metabolights/mars/ws3/submissions/",
153-
),
154-
"TOKEN": config.get(
155-
"metabolights",
156-
"production-token-url",
157-
fallback="https://www-test.ebi.ac.uk/metabolights/mars/ws3/auth/token",
158-
),
159-
},
160-
"BIOSAMPLES": {
161-
"SERVICE": config.get(
162-
"biosamples",
163-
"production-url",
164-
fallback="https://www.ebi.ac.uk/biosamples/samples/",
165-
),
166-
"SUBMISSION": config.get(
167-
"biosamples",
168-
"production-submission-url",
169-
fallback="https://www.ebi.ac.uk/biosamples/samples/",
170-
),
171-
},
172-
},
173-
}
33+
urls = TargetRepository.get_repository_urls_from_config(config)
17434

17535

17636
@click.group()
@@ -183,6 +43,10 @@
18343
@click.pass_context
18444
def cli(ctx, development):
18545
print_and_log("############# Welcome to the MARS CLI. #############")
46+
print_and_log(
47+
"Sensitive information might be dumped in the log files when setting the 'log_level' to DEBUG in the config file. Logging in debug mode should only be used for developing purpose a can implicate security issues if used in a production environment!",
48+
"debug",
49+
)
18650
print_and_log(
18751
f"Running in {'Development environment' if development else 'Production environment'}"
18852
)

‎mars-cli/mars_lib/ftp_upload.py

+5
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,12 @@ def upload(self, file_paths: List[Path], target_location: str = "/") -> bool:
3838
with PatchFTP_TLS() as ftps:
3939
ftps.context.set_ciphers("HIGH:!DH:!aNULL")
4040
ftps.connect(self.ftp_host, port=21, timeout=timeout)
41+
print_and_log(f"Connected to {self.ftp_host}", "debug")
4142
ftps.login(self.username, self.password)
43+
print_and_log(
44+
f"User '{self.username}' successfully logged in to {self.ftp_host}",
45+
"debug",
46+
)
4247
ftps.prot_p()
4348

4449
ftps.cwd(target_location)

0 commit comments

Comments
 (0)