diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.yml index 77c5fe5..a9bfcbd 100644 --- a/.github/workflows/build_package.yml +++ b/.github/workflows/build_package.yml @@ -15,11 +15,11 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Set up JDK 8 + - name: Set up JDK 17 uses: actions/setup-java@v3 with: distribution: 'zulu' - java-version: '8' + java-version: '17' - name: Load local Maven repository cache uses: actions/cache@v3 with: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1b1841d..9c095a7 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -38,11 +38,11 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Set up JDK 8 + - name: Set up JDK 17 uses: actions/setup-java@v3 with: distribution: 'zulu' - java-version: '8' + java-version: '17' settings-path: ${{ github.workspace }} - name: Load local Maven repository cache diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index ed38abf..6fba126 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -13,11 +13,11 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Set up JDK 8 + - name: Set up JDK 17 uses: actions/setup-java@v3 with: distribution: 'zulu' - java-version: '8' + java-version: '17' settings-path: ${{ github.workspace }} - name: Load local Maven repository cache diff --git a/.github/workflows/nexus-publish-snapshots.yml b/.github/workflows/nexus-publish-snapshots.yml index 9d98c35..bd76fde 100644 --- a/.github/workflows/nexus-publish-snapshots.yml +++ b/.github/workflows/nexus-publish-snapshots.yml @@ -17,11 +17,11 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Set up JDK 8 + - name: Set up JDK 17 uses: actions/setup-java@v3 with: distribution: 'zulu' - java-version: '8' + java-version: '17' settings-path: ${{ github.workspace }} - name: Load local Maven repository cache diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index a5e3281..2557820 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -15,11 +15,11 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Set up JDK 8 + - name: Set up JDK 17 uses: actions/setup-java@v3 with: distribution: 'zulu' - java-version: '8' + java-version: '17' settings-path: ${{ github.workspace }} - name: Load local Maven repository cache diff --git a/README.md b/README.md index 707261a..5a96753 100644 --- a/README.md +++ b/README.md @@ -16,19 +16,19 @@ -A client software written in Java for dataset downloads from QBiC's data management system openBIS (https://wiki-bsse.ethz.ch/display/bis/Home). +Client software for downloading datasets from QBiC's data management system openBIS (https://wiki-bsse.ethz.ch/display/bis/Home). -We are making use of the V3 API of openBIS (https://wiki-bsse.ethz.ch/display/openBISDoc1605/openBIS+V3+API) in order to interact with the data management system from command line, in order to provide a quick data retrieval on server or cluster resources, where the download via the qPortal is impractical. +We are making use of the V3 API of openBIS (https://wiki-bsse.ethz.ch/display/openBISDoc1605/openBIS+V3+API) in order to interact with the data management system from the command line, in order to provide a quick data retrieval on the server or cluster resources, where the download via the qPortal is impractical. ## How to run ### Download You can download the compiled and executable Java binaries as JAR of postman from the GitHub release page: https://github.com/qbicsoftware/postman-cli/releases. -If you want to build from source, checkout the commit of your choice and execute `mvn clean package`. We only recommend this if you are familiar with Java build systems though, as we cannot give you support here. In the normal case, the binary of a stable release is sufficient. +If you want to build from source, check out the commit of your choice and execute `mvn clean package`. We only recommend this if you are familiar with Java build systems though, as we cannot give you support here. In the normal case, the binary of a stable release is sufficient. ### Requirements -You need to have **Java JRE** or **JDK** installed (**openJDK** is fine), at least version 1.8 or 11. And the client's host must have allowance to connect to the server, which is determined by our firewall settings. If you are unsure, if your client is allowed to connect, contact us at support@qbic.zendesk.com. +You need to have **Java JRE** or **JDK** installed (**openJDK** is fine), at least version 17. And the client's host must have allowance to connect to the server, which is determined by our firewall settings. If you are unsure, if your client is allowed to connect, contact us at support@qbic.zendesk.com. ### Configuration @@ -37,250 +37,249 @@ Example: ```bash java -jar postman.jar list -u @path/to/config.txt ``` -The structure of the configuration file is: [-cliOption] [value] +The structure of the configuration file is: +```text +[-cliOption] [value] +``` For example: To set the ApplicationServerURL to another URL we have to use: -as [URL] To use our openBIS URL we write the following lines in the config file: (Anything beginning with '#' is a comment) ``` +# Config file for postman-cli +# Replace the values defined after the respective CLI parameters (e.g. -u) +# with your value of choice (e.g. qbcab01 as value for -u) -# Set the AS_URL (ApplicationServerURL) to the value defined below --as https://qbis.qbic.uni-tuebingen.de/openbis/openbis - -# The following config file options are currently supported: -# AS_URL (Application Server URL) --as -# DSS_URL (DataStore Server URL) --dss [,...] - +--suffix .txt,.fastq,.fastq.gz +-u qbc001 +--password:env MY_PASSWORD ``` -A default file is provided here: [default-config](https://github.com/qbicsoftware/postman-cli/blob/development/config.txt). If no config file is provided, postman uses the default values set in the PostmanCommandLineOptions class. - -If no config file or commandline option is provided, Postman will resort to the defaults set here: [Defaults](https://github.com/qbicsoftware/postman-cli/blob/development/src/main/java/life/qbic/io/commandline/PostmanCommandLineOptions.java). -Hence, the default AS is set to: `https://qbis.qbic.uni-tuebingen.de/openbis/openbis` -and the DSS defaults to: `https://qbis.qbic.uni-tuebingen.de/datastore_server` and `https://qbis.qbic.uni-tuebingen.de/datastore_server2` +A default file is provided here: [default-config](https://github.com/qbicsoftware/postman-cli/blob/development/config.txt). ## How to use ### Options -Just execute postman with `java -jar postman-cli.jar` or `java -jar postman.jar -h` to get an overview of the options available for all subcommands: -```bash +Just execute the following command to get an overview of the available options. +```bash +java -jar postman-cli.jar -h -~$ java -jar postman.jar -Usage: postman-cli [-h] [-as=] [-dss=] [-f=] - [-p=] -u= [-s=[, - ...]]... [SAMPLE_ID...] [COMMAND] +Usage: postman-cli [-hV] COMMAND Description: -A client software for dataset downloads from QBiC\'s data management system openBIS. - -Parameters: - [SAMPLE_ID...] one or more QBiC sample ids +A software client for downloading data from QBiC. Options: - -V, --version print version information - -u, --user= openBIS user name - -p, --env-password= provide the name of an environment variable to read - in the password from - -as, -as_url= ApplicationServer URL - -dss, --dss_url=[,...] DataStoreServer URLs. Specifies the - data store servers where data can be found. - -f, --file= a file with line-separated list of QBiC sample ids - -s, --suffix=[,...] only include files ending with one of these suffixes - -h, --help display a help message and exit + -h, --help Show this help message and exit. + -V, --version Print version information and exit. Commands: - download Download data from OpenBis - list List all available datasets for the given identifiers with additional metadata + download Download data from QBiC. + list lists all the datasets found for the given identifiers Optional: specify a config file by running postman with '@/path/to/config.txt'. +A detailed documentation can be found at +https://github.com/qbicsoftware/postman-cli#readme. +``` +### Printed file sizes +File sizes printed by postman Kilobyte, Megabyte, Gigabyte, Terabyte and Petabyte use base 2. -``` -To get only the options for one of the two subcommands, execute postman with `java -jar postman.jar download -h` or `java -jar postman.jar list -h`. -#### Provide a file with several QBiC IDs -In order to download datasets from several samples at once, you can provide a manifest file consisting of multiple, line-separated, QBiC IDs. -Hand it to postman with the `-f` option. - -For example: - -```bash -java -jar postman.jar download -s fastq.gz -f myids.txt -u -``` - -with `myids.txt` like: +### How to specify your log directory +By default, postman will create log files in a directory `./logs/` in your working directory. +You can specify where logs are written by setting the system property `log.path` to the desired directory. -``` -QTEST001AE -QTEST002BD -... -``` +### How to provide your credentials +After gaining access by applying through support@qbic.zendesk.com, you can log in using your credentials. +The username is provided by us or if you have a uni-tuebingen account by the university of Tuebingen. -postman will automatically iterate over the IDs. +**Provide your username:** +Use the option `-u / --user` to provide us with your username. -#### Filter for file suffix +**Provide your password:** -For example filter for fastq.gz files only: - -``` -java -jar postman.jar download -s fastq.gz -u QMFKD003AG -java -jar postman.jar list -s fastq.gz -u QMFKD003AG +Please never send your password over email. We will never ask you for it!
+When using the application, you can either: +1. enter your password interactively `--password` +2. enter the name of a system property containing your password `--password:prop my.awesome.property` +```bash +java -jar -Dmy.awesome.property=ABCDEFG postman.jar -u qbc001a --password:prop my.awesome.property ``` - -For example filter for fastq.gz and fastq files only: +3. enter the name of an environment variable containing your password `--password:env MY_PASSWORD` +```bash +MY_PASSWORD=ABCDEFG java -jar postman.jar -u qbc001a --password:env MY_PASSWORD ``` -java -jar postman.jar download -s fastq,fastq.gz -u QMFKD003AG -java -jar postman.jar list -s fastq,fastq.gz -u QMFKD003AG +### How to provide QBiC identifiers +To specify which data you want to list or download, you need to provide us with QBiC identifiers. +A QBiC project identifier begins with `Q` followed by four characters (`QTEST`). QBiC sample identifiers contain their project identifier. +You can provide identifiers either using the command line directly: +```bash +java -jar postman.jar QSTTSABCD01 QSTTSABCD02 QSTTSABCDE4 NGSQSTTS0012 "QTEST*" ``` +Please note: When you use the `*` character to search for all files in a project, escape your identifier using quotation marks. -#### Provide your password with an environment variable -If you do not want to provide your password manually every time, you can use the `-p` option instead. -Set a new environment variable with your password as a value. Then, you can execute postman with `-p ` and postman will automatically read in your password from the environment variable. - -###### Setting environment variable +**Provide identifiers using a file:** -**Windows** -To set your environment variable for the current cmd session, you can use this command: -```bash -~$ set VARIABLE_NAME=password +You can provide identifiers using a file containing the identifiers. Lines starting with `#` are ignored. +```text +# all files for the project +QTEST* +# all files associated with these samples +QSTTS001AB +QSTTS002BC ``` -Make sure to not put a space before and after the "=" sign. -This command will not define a permanent environment variable. It will only be accessible during the current cmd session. -If you want to assign it permanently, you have to add it via the advanced settings of the Control Panel. - -**MacOs/Linux** - To set your environment variable for the current cmd session, you can use this command: -```bash -~$ export VARIABLE_NAME=password +```bash +java -jar postman.jar -f myids.txt ``` -Make sure to not put a space before and after the "=" sign. -This command will not define a permanent environment variable. It will only be accessible during the current cmd session. -If you want to assign it permanently, you have to add the export command to your bash shells startup script. -### Subcommands -There are two available subcommands to use: `download` and `list`. -It is **always required** to specify one of these subcommands. +### How to filter files by suffix +Both the `download` and the `list` command allow you to filter files by their suffix. The suffixes provided are not case-sensitive. +`.TXT` and `.txt` will have the same effect. +Multiple suffixes can be provided separated by a comma. A suffix does not have to be the file ending but can be any substring from the end of a file's name. -### `list` -Provide this subcommand if you only want to get information about the given samples. **Nothing will be downloaded**. - -For each Sample ID, all available datasets and the files they contain will be listed as output on the terminal. -For all files size and name are provided. Additionally, registration date, size and source of each dataset are displayed. - -The easiest way to access the information about a sample is to execute postman with the subcommand `list` together with the QBiC ID for that sample and your username (same as the one you use for the qPortal): +If you only want to download `fastq` and `fastq.gz` files you can run postman with ```bash -~$ java -jar postman.jar list -u -``` -Postman will prompt you for your password, which is the password from your QBiC user account. -After a successful authentication, a possible result can look like this: -```bash -[bbbfs01@u-003-ncmu03 ~]$ java -jar postman.jar list -u bbbfs01 NGSQSTTS016A8 NGSQSTTS019AW -Provide password for user 'bbbfs01': - -Number of datasets found for identifier NGSQSTTS016A8 : 1 -# Dataset NGSQSTTS016A8 (20211215154407692-131872) -# Source QSTTS016A8 -# Registration 2021-12-15T02:44:07Z -# Size 2.10MB -1.05MB testfile1 -1.05MB testfile2 - -Number of datasets found for identifier NGSQSTTS019AW : 1 -# Dataset NGSQSTTS019AW (20211215154408961-131875) -# Source QSTTS019AW -# Registration 2021-12-15T02:44:09Z -# Size 1.05MB -1.05MB testfile +java -jar postman.jar -s .fastq,.fastq.gz ``` +## `list` +```txt +Usage: postman-cli list [-hV] [--exact-filesize] [--with-checksum] + [--without-header] [--format=] -u= + [-s=[,...]]... (--password: + env= | --password: + prop= | --password) (-f= | + SAMPLE_IDENTIFIER...) + +Description: +lists all the datasets found for the given identifiers + +Parameters: + SAMPLE_IDENTIFIER... one or more QBiC sample identifiers + +Options: + --password:env= + provide the name of an environment variable to + read in your password from + --password:prop= + provide the name of a system property to read in + your password from + --password please provide your password + -u, --user= openBIS user name + -f, --file= a file with line-separated list of QBiC sample ids + -s, --suffix=[,...] + only include files ending with one of these + (case-insensitive) suffixes + --with-checksum list the crc32 checksum for each file + --exact-filesize use exact byte count instead of unit suffixes: + Byte, Kilobyte, Megabyte, Gigabyte, Terabyte and + Petabyte + --format= The format to list files in. Case-insensitive. + Possible values: LEGACY, TSV + Default: LEGACY + --without-header remove the header line from the output. Only takes + effect for tabular output formats. + -h, --help Show this help message and exit. + -V, --version Print version information and exit. -##### Options -When using the subcommand `list`, there are further options available: +Optional: specify a config file by running postman with '@/path/to/config.txt'. +A detailed documentation can be found at +https://github.com/qbicsoftware/postman-cli#readme. ``` - --with-checksum print the crc32 checksum as second column. - Default: false +The `list` command comes with some special options. You can change how your output looks by +1. listing the exact number of bytes for each file `--exact-filesize` +2. removing the header from the tabular output `--without-header` +3. listing the crc32 checksum for every file `--with-checksum` +4. specifying the output format. + +#### `TSV` format +```text +Dataset Source Registration Size File +NGSQSTTS015A0 (20211026111452695-847006) NGSQSTTS015A0 2021-10-26T09:14:53.143812Z 1,00 B QSTTS015A0_awesome_genome.fastq/my_genome.fastq ``` -```bash -[bbbfs01@u-003-ncmu03 ~]$ java -jar postman.jar list --with-checksum -u bbbfs01 NGSQSTTS016A8 NGSQSTTS019AW -Provide password for user 'bbbfs01': - -Number of datasets found for identifier NGSQSTTS016A8 : 1 -# Dataset NGSQSTTS016A8 (20211215154407692-131872) -# Source QSTTS016A8 -# Registration 2021-12-15T02:44:07Z -# Size 2.10MB -1.05MB 83d57075 testfile1 -1.05MB dff8e2e2 testfile2 - -Number of datasets found for identifier NGSQSTTS019AW : 1 -# Dataset NGSQSTTS019AW (20211215154408961-131875) -# Source QSTTS019AW -# Registration 2021-12-15T02:44:09Z -# Size 1.05MB -1.05MB 0a84cf87 testfile +```text +Dataset Source Registration Size CRC32 File +NGSQSTTS015A0 (20211026111452695-847006) NGSQSTTS015A0 2021-10-26T09:14:53.143812Z 1,00 B fa0a3f23 QSTTS015A0_awesome_genome.fastq/my_genome.fastq ``` - -##### Dataset vs. File -The structure of samples can sometimes seem a little confusing. To clarify the difference of a dataset and a file, you can take a look at the example result above. -- A sample can contain several datasets. -- All of these datasets can contain several files. -→ a dataset is a collection of files and does not contain any data itself. -→ the files are the ones that contain data, depending on the file format. -Another thing to think of is that samples as well as datasets can be empty. +#### `LEGACY` format +```text +# Dataset NGSQSTTS015A0 (20211026111452695-847006) +# Source NGSQSTTS015A0 +# Registration 2021-10-26T09:14:53.143812Z +# Size 1,00 B +1,00 B my_genome.fastq +``` +```text +# Dataset NGSQSTTS015A0 (20211026111452695-847006) +# Source NGSQSTTS015A0 +# Registration 2021-10-26T09:14:53.143812Z +# Size 1,00 B +1,00 B fa0a3f23 my_genome.fastq +``` +## `download` -### `download` -The other available subcommand is `download`. -With this command, existing files will be downloaded for the provided sample IDs. +```txt +Usage: postman-cli download [-hV] [-o=] -u= [-s=[, + ...]]... (--password: + env= | --password: + prop= | --password) (-f= + | SAMPLE_IDENTIFIER...) -The simplest scenario is, that you want to download a dataset/datasets from a sample. Just provide the QBiC ID for that sample and your username (same as the one you use for the qPortal): -```bash -~$ java -jar postman.jar download -u -``` -Postman will prompt you for your password, which is the password from your QBiC user account. +Description: +Download data from QBiC. -After you have provided your password and authenticate successfully, postman tries to download all datasets that are registered for that given sample ID and downloads them to the current working directory: +Parameters: + SAMPLE_IDENTIFIER... one or more QBiC sample identifiers -```bash -[bbbfs01@u-003-ncmu03 ~]$ java -jar postman.jar download -u bbbfs01 QMFKD003AG -Provide password for user 'bbbfs01': - -12:32:02.038 [main] INFO life.qbic.App - OpenBis login returned with 0 -12:32:02.043 [main] INFO life.qbic.App - Connection to openBIS was successful. -12:32:02.043 [main] INFO life.qbic.App - 1 provided openBIS identifiers have been found: [QMFKD003AG] -12:32:02.044 [main] INFO life.qbic.App - Downloading files for provided identifier QMFKD003AG -12:32:02.278 [main] INFO life.qbic.App - Number of data sets found: 2 -12:32:02.279 [main] INFO life.qbic.App - Initialize download ... -QMFKD003AG_SRR099967_1.filt.fastq.gz [### ] 0.38/7.94 Gb -``` +Options: + --password:env= + provide the name of an environment variable to + read in your password from + --password:prop= + provide the name of a system property to read in + your password from + --password please provide your password + -u, --user= openBIS user name + -f, --file= a file with line-separated list of QBiC sample ids + -s, --suffix=[,...] + only include files ending with one of these + (case-insensitive) suffixes + -o, --output-dir= + specify where to write the downloaded data + -h, --help Show this help message and exit. + -V, --version Print version information and exit. -##### Options -When using the subcommand `download`, there are further options available: -``` - -c, --conserve Conserve the file path structure during download - -b, --buffer-size= Dataset download performance can be improved by increasing this value with a multiple of 1024 (default). - Only change this if you know what you are doing. - -o,--output-dir= Provide an already existing path where you want your data to be downloaded to +Optional: specify a config file by running postman with '@/path/to/config.txt'. +A detailed documentation can be found at +https://github.com/qbicsoftware/postman-cli#readme. ``` -###### Download all data of a project +The `download` command allows you to download data from our storage to your machine. -If you want to download all datasets for a given project id, you can use the wildcard symbol `*` and have to specify the project code inside of quotation marks: -```bash -~$ java -jar postman.jar download -u "*" -``` +Use the `--output-dir` option to specify a location on your client the location will be interpreted relative to your working directory. ##### File integrity check -Postman computes the CRC32 checksum for all input streams using the native Java utility class [CRC32](https://docs.oracle.com/javase/8/docs/api/java/util/zip/CRC32.html). Postman favors [`CheckedInputStream`](https://docs.oracle.com/javase/7/docs/api/java/util/zip/CheckedInputStream.html) +Postman computes the CRC32 checksum for all input streams using the native Java utility class [CRC32](https://docs.oracle.com/javase/8/docs/api/java/util/zip/CRC32.html). Postman favours [`CheckedInputStream`](https://docs.oracle.com/javase/7/docs/api/java/util/zip/CheckedInputStream.html) over the traditional InputStream, and promotes the CRC checksum computation. -The expected CRC32 checksums are derived via the openBIS API and compared with the computed ones after the download. - -Postman writes two additional summary files for that in the `logs` folder of the current working directory: `summary_valid_files.txt` and `summary_invalid_files.txt`. -They contain the computed and expected checksum as hex string plus the file path of the recorded file: +Computed CRC32 checksums are compared with CRC32 checksums stored on our servers. +When the checksum does not match, then the download failed. Each download is attempted multiple times. +When all attempts fail, the mismatching checksum is recorded in your log directory in the `checksum-mismatch.log` file. +The `checksum-mismatch.log` file contains one line for each file that was not downloaded. ``` // values are tab separated ``` In addition, Postman writes the CRC32 checksum in an additional file `.crc32` and stores it together with the according file. + +#### Advanced Options +##### `postman` +* `-Dlog.path`: provide the log directory +* `-Dlog.level`: provide the log level to use for logging +* `--source-sample-type `: specify which sample type to consider as source sample type. +* `--server-timeout `: the server timeout in milliseconds + +##### `download` +* `--download-attempts ` provide the maximal amount attempted downloads +* `--buffer-size ` provide a custom buffer size. Please only specify values that are a multiple of `1024`. diff --git a/config.txt b/config.txt index 1241e56..970a2aa 100644 --- a/config.txt +++ b/config.txt @@ -1,10 +1,9 @@ # Config file for postman-cli -# Replace the values defined after the respective CLI parameters (e.g. -as) -# with your value of choice (e.g. https://qbis.qbic.uni-tuebingen.de/openbis/openbis as value for -as) +# Replace the values defined after the respective CLI parameters # Set the AS_URL (ApplicationServerURL) to the value defined below -# -as https://qbis.qbic.uni-tuebingen.de/openbis/openbis --as https://openbis1605test.am10.uni-tuebingen.de/openbis/openbis +--application-server https://qbis.qbic.uni-tuebingen.de/openbis/openbis # Set the DSS_URL (DataStoreServerURL) to the value defined below --dss https://qbis.qbic.uni-tuebingen.de/datastore_server,https://qbis.qbic.uni-tuebingen.de/datastore_server2 +--datastore_server https://qbis.qbic.uni-tuebingen.de/datastore_server +--datastore_server https://qbis.qbic.uni-tuebingen.de/datastore_server2 diff --git a/pom.xml b/pom.xml index c1e2e83..144361f 100644 --- a/pom.xml +++ b/pom.xml @@ -12,11 +12,11 @@ jar - 1.8 - 1.8 + 17 + 17 UTF-8 - 3.0.15 - 2.17.1 + 4.0.12 + 2.20.0 @@ -60,12 +60,21 @@ QBiC Releases https://qbic-repo.qbic.uni-tuebingen.de/repository/maven-releases + + - vaadin-addons - Vaadin Addons - https://maven.vaadin.com/vaadin-addons + true + nexus-releases + QBiC Releases + https://qbic-repo.qbic.uni-tuebingen.de/repository/maven-releases - + + false + nexus-snapshots + QBiC Snapshots + https://qbic-repo.qbic.uni-tuebingen.de/repository/maven-snapshots + + @@ -77,20 +86,13 @@ info.picocli picocli - 4.6.2 - - - jline - jline - 2.14.6 + 4.7.4 - - org.codehaus.groovy + org.apache.groovy groovy ${groovy.version} - indy @@ -112,13 +114,17 @@ 18.06.2 - junit - junit - 4.13.2 + org.junit.jupiter + junit-jupiter + 5.9.3 test + + org.spockframework + spock-core + 2.4-M1-groovy-4.0 + - @@ -147,7 +153,7 @@ org.codehaus.gmavenplus gmavenplus-plugin - 1.13.1 + 3.0.0 default @@ -179,27 +185,35 @@ + - org.apache.maven.plugins maven-surefire-plugin - 3.0.0-M3 + 3.1.2 + false **/*IntegrationTest.* - **/*Test.* - **/*Specification.* - **/*Spec.* - - target/jacoco.exec - + + **/*Test + **/*Specification + **/*Spec + + + false + 3.1 + false + true + true + true + org.apache.maven.plugins maven-failsafe-plugin - 3.0.0-M3 + 3.1.2 @@ -219,18 +233,4 @@ - - - true - nexus-releases - QBiC Releases - https://qbic-repo.qbic.uni-tuebingen.de/repository/maven-releases - - - false - nexus-snapshots - QBiC Snapshots - https://qbic-repo.qbic.uni-tuebingen.de/repository/maven-snapshots - - diff --git a/src/main/groovy/life/qbic/ChecksumReporter.groovy b/src/main/groovy/life/qbic/ChecksumReporter.groovy deleted file mode 100644 index 8dc2701..0000000 --- a/src/main/groovy/life/qbic/ChecksumReporter.groovy +++ /dev/null @@ -1,40 +0,0 @@ -package life.qbic - -import java.nio.file.Path - -/** - * This interface can be used if you want to provide - * functionality to report matching, mismatching checksums as well as storing a - * checksum for downloaded files. - * - * @author Sven Fillinger - * @since 0.4.0 - */ -interface ChecksumReporter { - - /** - * Writes a matching checksum into a summary report for valid checksums. - * - * @param expectedChecksum The checksum that was expected from the file. - * @param computedChecksum The checksum that was calculated from the file. - * @param fileLocation The file location for which the checksum was compared. - */ - void reportMatchingChecksum(String expectedChecksum, String computedChecksum, URL fileLocation) - - /** - * Writes a mismatching checksum into a summary report for mismatching checksums. - * @param expectedChecksum The checksum that was expected from the file. - * @param computedChecksum The checksum that was calculated from the file. - * @param fileLocation The file location for which the checksum was compared. - */ - void reportMismatchingChecksum(String expectedChecksum, String computedChecksum, URL fileLocation) - - /** - * Writes String content to a file with a provided path. - * - * @param filePath The path of the file for which the checksum was calculated. - * @param content The calculated checksum. - */ - void storeChecksum(Path filePath, String checksum) - -} diff --git a/src/main/groovy/life/qbic/DownloadException.groovy b/src/main/groovy/life/qbic/DownloadException.groovy deleted file mode 100644 index 910bf13..0000000 --- a/src/main/groovy/life/qbic/DownloadException.groovy +++ /dev/null @@ -1,28 +0,0 @@ -package life.qbic - -/** - * Thrown, when a resource download attempt failed. - * - * This includes: - * - Stream interruption - * - Remote server exceptions - * - Local I/O exceptions - * - * @author Sven Fillinger - * @since 0.4.0 - */ -class DownloadException extends RuntimeException { - - DownloadException() { - super() - } - - DownloadException(String m) { - super(m) - } - - DownloadException(Throwable cause) { - super(cause) - } - -} diff --git a/src/main/groovy/life/qbic/DownloadRequest.groovy b/src/main/groovy/life/qbic/DownloadRequest.groovy deleted file mode 100644 index ac081a6..0000000 --- a/src/main/groovy/life/qbic/DownloadRequest.groovy +++ /dev/null @@ -1,60 +0,0 @@ -package life.qbic - -import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile - -import java.nio.file.Path - -/** - * Contains information about a download request for openBIS - * data set files. - * - * Allows to add and access data set files to and from a request - * Allows for quicks access of CRC32 checksums of corresponding data set files. - * - * @author Sven Fillinger - * @since 0.4.0 - */ -class DownloadRequest { - private final List dataSetFiles - private final Path prefix - private final long retries - - - /** - * Download request constructor with a provided list of data set files and a configured - * number of retries on failure. - * - * @param datasetSampleCode the code of the sample to which the dataset was attached - * @param dataSetFiles the files to download - * @param numberRetries The number of retries. Must be >=1, else it will be set to 1 - */ - DownloadRequest(Path prefix, List dataSetFiles, long numberRetries) { - Objects.requireNonNull(prefix, "prefix must not be null") - Objects.requireNonNull(dataSetFiles, "files must not be null") - - this.dataSetFiles = new ArrayList<>() - this.dataSetFiles.addAll(dataSetFiles) - this.retries = numberRetries >= 1 ? numberRetries : 1 - this.prefix = prefix - } - - /** - * Returns the max number of attempts for the download request. - * @return The number of attempts to perform, if the download fails - */ - long getMaxNumberOfAttempts() { - return retries - } - - /** - * Returns a shallow copy of the data set file list from the download request. - * @return A list of all requested data set files. - */ - List getFiles() { - dataSetFiles.asUnmodifiable() - } - - Path getPrefix() { - return prefix - } -} diff --git a/src/main/groovy/life/qbic/FileSystemWriter.groovy b/src/main/groovy/life/qbic/FileSystemWriter.groovy deleted file mode 100644 index fda1632..0000000 --- a/src/main/groovy/life/qbic/FileSystemWriter.groovy +++ /dev/null @@ -1,72 +0,0 @@ -package life.qbic - -import java.nio.file.Path -import java.nio.file.Paths - -/** - * File system implementation of the ChecksumWriter interface. - * - * Provides methods to write matching checksums and failed checksums - * into different summary files on the local file system. - * - * @author: Sven Fillinger - */ -class FileSystemWriter implements ChecksumReporter { - - /** - * File that stores the summary report content for valid checksums. - */ - final private File matchingSummaryFile - - /** - * File that stores the summary report content for invalid checksums. - */ - final private File failureSummaryFile - - - /** - * FileSystemWriter constructor with the paths for the summary files. * - * - * @param matchingSummarFile The path where to write the matching checksum summary - * @param failureSummaryFile The path where to write the failed checksum summary - */ - FileSystemWriter(Path matchingSummaryFile, Path failureSummaryFile) { - this.matchingSummaryFile = new File(matchingSummaryFile.toString()) - this.failureSummaryFile = new File(failureSummaryFile.toString()) - } - - /** - * {@inheritDoc} - */ - @Override - void reportMatchingChecksum(String expectedChecksum, String computedChecksum, URL fileLocation) { - def content = "$expectedChecksum\t$computedChecksum\t${Paths.get(fileLocation.toURI())}\n" - this.matchingSummaryFile.append(content, "UTF-8") - } - - /** - * {@inheritDoc} - */ - @Override - void reportMismatchingChecksum(String expectedChecksum, String computedChecksum, URL fileLocation) { - def content = "$expectedChecksum\t$computedChecksum\t${Paths.get(fileLocation.toURI())}\n" - this.failureSummaryFile.append(content, "UTF-8") - } - - /** - * {@inheritDoc} - */ - @Override - void storeChecksum(Path filePath, String checksum) { - def newFile = new File(filePath.toString() + ".crc32") - if (!newFile.createNewFile()) { - //file exists or could not be created - if (!newFile.exists()) { - throw new IOException("The file " + newFile.getAbsoluteFile() + " could not be created.") - } - } - newFile.withWriter { - it.write(checksum + "\t" + filePath.getFileName()) - } - } -} diff --git a/src/main/groovy/life/qbic/QbicDataLoaderRegexUtil.groovy b/src/main/groovy/life/qbic/QbicDataLoaderRegexUtil.groovy deleted file mode 100644 index 1dfea70..0000000 --- a/src/main/groovy/life/qbic/QbicDataLoaderRegexUtil.groovy +++ /dev/null @@ -1,54 +0,0 @@ -package life.qbic - -import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.search.SearchResult -import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet -import ch.ethz.sis.openbis.generic.dssapi.v3.IDataStoreServerApi -import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile -import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.fetchoptions.DataSetFileFetchOptions -import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.search.DataSetFileSearchCriteria - -class QbicDataLoaderRegexUtil { - - /** - * Using dollar slashy regex of groovy to match all provided regexes to filter the dataset files - * - * @param regexPatterns - * @param allDatasets - * @param dataStoreServer - * @param sessionToken - * @return all fileIDs which are forwarded to download - */ - static List findAllRegexFilteredIDsGroovy(List regexPatterns, - List allDatasets, - IDataStoreServerApi dataStoreServer, - String sessionToken) { - List allFileIDs = new ArrayList<>() - - for (DataSet ds : allDatasets) { - // we cannot access the files directly of the datasets -> we need to query for the files first using the datasetID - DataSetFileSearchCriteria criteria = new DataSetFileSearchCriteria() - criteria.withDataSet().withCode().thatEquals(ds.getCode()) - SearchResult result = dataStoreServer.searchFiles(sessionToken, criteria, new DataSetFileFetchOptions()) - List files = result.getObjects() - - List filesFiltered = new ArrayList<>() - - // remove everything that doesn't match the regex -> only add if regex matches - for (DataSetFile file : files) { - for (String regex : regexPatterns) { - def fullRegex = $/$regex/$ - def matched = file.getPermId().toString() =~ fullRegex - - if (matched) { - filesFiltered.add(file) - } - } - } - - allFileIDs.addAll(filesFiltered) - } - - return allFileIDs - } - -} diff --git a/src/main/java/life/qbic/App.java b/src/main/java/life/qbic/App.java index e1206b8..13d9ba3 100644 --- a/src/main/java/life/qbic/App.java +++ b/src/main/java/life/qbic/App.java @@ -1,19 +1,11 @@ package life.qbic; -import life.qbic.io.commandline.OpenBISPasswordParser; -import life.qbic.io.commandline.PostmanCommandLineOptions; -import life.qbic.model.Configuration; -import life.qbic.model.download.Authentication; -import life.qbic.model.download.AuthenticationException; -import life.qbic.model.download.ConnectionException; +import java.util.Arrays; +import life.qbic.qpostman.common.PostmanCommand; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import picocli.CommandLine; -import java.io.File; -import java.util.Arrays; -import java.util.Optional; - /** * postman for staging data from openBIS */ @@ -23,76 +15,9 @@ public class App { public static void main(String[] args) { LOG.debug("command line arguments: " + Arrays.deepToString(args)); - CommandLine cmd = new CommandLine(new PostmanCommandLineOptions()); + CommandLine cmd = new CommandLine(new PostmanCommand()); int exitCode = cmd.execute(args); System.exit(exitCode); } - /** - * checks if the commandline parameter for reading out the password from the environment variable - * is correctly provided - */ - private static Boolean isNotNullOrEmpty(String envVariableCommandLineParameter) { - return envVariableCommandLineParameter != null && !envVariableCommandLineParameter.isEmpty(); - } - - /** - * Logs into OpenBIS asks for and verifies password. - * - * @return An instance of the Authentication class. - */ - public static Authentication loginToOpenBIS( - String passwordEnvVariable, String user, String as_url) { - - String password; - if (isNotNullOrEmpty(passwordEnvVariable)) { - Optional envPassword = OpenBISPasswordParser.readPasswordFromEnvVariable( - passwordEnvVariable); - if (!envPassword.isPresent()) { - System.out.println( - "No environment variable named " + passwordEnvVariable - + " was found"); - LOG.info( - String.format("Please provide a password for user '%s':", user)); - } - password = envPassword.orElseGet(OpenBISPasswordParser::readPasswordFromConsole); - } else { - LOG.info( - String.format("Please provide a password for user '%s':", user)); - password = OpenBISPasswordParser.readPasswordFromConsole(); - } - - if (password.isEmpty()) { - LOG.error("You need to provide a password."); - System.exit(1); - } - - // Ensure 'logs' folder is created - File logFolder = new File(Configuration.LOG_PATH.toAbsolutePath().toString()); - if (!logFolder.exists()) { - boolean logFolderCreated = logFolder.mkdirs(); - if (!logFolderCreated) { - LOG.error("Could not create log folder '" + logFolder.getAbsolutePath() + "'"); - System.exit(1); - } - } - - Authentication authentication = - new Authentication( - user, - password, - as_url); - try { - authentication.login(); - } catch (ConnectionException e) { - LOG.error(e.getMessage(), e); - LOG.error("Could not connect to QBiC's data source. Have you requested access to the " - + "server? If not please write to support@qbic.zendesk.com"); - System.exit(1); - } catch (AuthenticationException e) { - LOG.error(e.getMessage()); - System.exit(1); - } - return authentication; - } } diff --git a/src/main/java/life/qbic/io/commandline/OpenBISPasswordParser.java b/src/main/java/life/qbic/io/commandline/OpenBISPasswordParser.java deleted file mode 100644 index 55923f7..0000000 --- a/src/main/java/life/qbic/io/commandline/OpenBISPasswordParser.java +++ /dev/null @@ -1,30 +0,0 @@ -package life.qbic.io.commandline; - -import java.io.Console; -import java.util.Optional; - -public class OpenBISPasswordParser { - - - /** - * Retrieve the password from the system console - * - * @return the password read from the system console input - */ - public static String readPasswordFromConsole() { - Console console = System.console(); - char[] passwordChars = console.readPassword(); - return String.valueOf(passwordChars); - } - - /** - * @param variableName Name of given environment variable - * @return the password read from the environment variable - */ - public static Optional readPasswordFromEnvVariable(String variableName) { - - return Optional.ofNullable(System.getenv(variableName)); - - } - -} diff --git a/src/main/java/life/qbic/io/commandline/PostmanCommandLineOptions.java b/src/main/java/life/qbic/io/commandline/PostmanCommandLineOptions.java deleted file mode 100644 index 71e5a2e..0000000 --- a/src/main/java/life/qbic/io/commandline/PostmanCommandLineOptions.java +++ /dev/null @@ -1,182 +0,0 @@ -package life.qbic.io.commandline; - -import static java.util.Objects.isNull; -import static java.util.Objects.nonNull; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import life.qbic.App; -import life.qbic.io.parser.IdentifierParser; -import life.qbic.model.download.Authentication; -import life.qbic.model.download.QbicDataDisplay; -import life.qbic.model.download.QbicDataDownloader; -import life.qbic.model.download.QbicDataDownloader.DownloadResponse; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import picocli.CommandLine; -import picocli.CommandLine.Command; -import picocli.CommandLine.Help.Visibility; -import picocli.CommandLine.Option; -import picocli.CommandLine.Parameters; - -// main command with format specifiers for the usage help message -@Command(name = "postman-cli", - versionProvider = ManifestVersionProvider.class, - footer = "Optional: specify a config file by running postman with '@/path/to/config.txt'. Details can be found in the README.", - description = "A client software for dataset downloads from QBiC's data management system openBIS.", - usageHelpAutoWidth = true, - sortOptions = false, - descriptionHeading = "%nDescription:%n", - parameterListHeading = "%nParameters:%n", - optionListHeading = "%nOptions:%n", - commandListHeading = "%nCommands:%n", - footerHeading = "%n") - -public class PostmanCommandLineOptions { - private static final Logger LOG = LogManager.getLogger(QbicDataDownloader.class); - - @Option(names = {"-V", "--version"}, - versionHelp = true, - description = "print version information", - scope = CommandLine.ScopeType.INHERIT) - boolean versionRequested; - - - //parameters to format the help message - @Command(name = "download", - versionProvider = ManifestVersionProvider.class, - description = "Download data from OpenBis", - usageHelpAutoWidth = true, - sortOptions = false, - descriptionHeading = "%nDescription: ", - parameterListHeading = "%nParameters:%n", - optionListHeading = "%nOptions:%n", - footerHeading = "%n") - void download( - @Option(names = {"-c", "--conserve"}, - description = "Conserve the file path structure during download") boolean conservePath, - @Option(names = {"-b", "--buffer-size"}, defaultValue = "1", - description = - "dataset download performance can be improved by increasing this value with a multiple of 1024 (default)." - + " Only change this if you know what you are doing.") int bufferMultiplier, - @Option( - names = {"-o", "--output-dir"}, - description = "provide the path to an existing directory where you want to download your data to") String outputPath) - throws IOException { - Authentication authentication = App.loginToOpenBIS(passwordEnvVariable, user, as_url); - - QbicDataDownloader qbicDataDownloader = - new QbicDataDownloader( - as_url, - dss_urls, - bufferMultiplier * 1024, - conservePath, - authentication.getSessionToken(), - outputPath); - ids = verifyProvidedIdentifiers(); - DownloadResponse downloadResponse = qbicDataDownloader.downloadForIds(ids, suffixes); - LOG.info("Done"); - if (downloadResponse.containsFailure()) { - LOG.warn(String.format("Failed to download %s out of %s files", - downloadResponse.failureCount(), downloadResponse.fileCount())); - System.exit(1); - } - } - - @Command(name = "list", - description = "lists all the datasets found for the given identifiers", - usageHelpAutoWidth = true, - sortOptions = false, - descriptionHeading = "%nDescription: ", - parameterListHeading = "%nParameters:%n", - optionListHeading = "%nOptions:%n", - footerHeading = "%n") - void listDatasets( - @Option(names = "--with-checksum", defaultValue = "false", description = "print the crc32 checksum as second column.", showDefaultValue = Visibility.ALWAYS) - boolean withChecksum) - throws IOException { - - Authentication authentication = App.loginToOpenBIS(passwordEnvVariable, user, as_url); - QbicDataDisplay qbicDataDisplay = new QbicDataDisplay(as_url, - dss_urls, - authentication.getSessionToken(), withChecksum); - ids = verifyProvidedIdentifiers(); - qbicDataDisplay.getInformation(ids, suffixes); - } - - @Parameters(paramLabel = "SAMPLE_ID", description = "one or more QBiC sample ids", scope = CommandLine.ScopeType.INHERIT) - public List ids; - - @Option( - names = {"-u", "--user"}, - required = true, - description = "openBIS user name", - scope = CommandLine.ScopeType.INHERIT) - public String user; - - @Option( - names = {"-p", "--env-password"}, - description = "provide the name of an environment variable to read in the password from", - scope = CommandLine.ScopeType.INHERIT) - public String passwordEnvVariable; - - @Option( - names = {"-as", "-as_url"}, - description = "ApplicationServer URL", - scope = CommandLine.ScopeType.INHERIT) - public String as_url = "https://qbis.qbic.uni-tuebingen.de/openbis/openbis"; - - @Option( - names = {"-dss", "--dss_url"}, - split = ",", - paramLabel = "", - description = "DataStoreServer URLs. Specifies the data store servers where data can be found.", - scope = CommandLine.ScopeType.INHERIT) - public List dss_urls = new ArrayList(){{ - add("https://qbis.qbic.uni-tuebingen.de/datastore_server"); - add("https://qbis.qbic.uni-tuebingen.de/datastore_server2"); - }}; - - @Option( - names = {"-f", "--file"}, - description = "a file with line-separated list of QBiC sample ids", - scope = CommandLine.ScopeType.INHERIT) - public Path filePath; - - @Option(names = {"-s", "--suffix"}, - split = ",", - description= "only include files ending with one of these (case-insensitive) suffixes", - paramLabel = "", - scope = CommandLine.ScopeType.INHERIT) - public List suffixes; - - - @Option( - names = {"-h", "--help"}, - usageHelp = true, - description = "display a help message and exit", - scope = CommandLine.ScopeType.INHERIT) - public boolean helpRequested = false; - - /** - * @return sample identifiers - * @throws IOException if no ids or command line argument ids & file were provided - */ - private List verifyProvidedIdentifiers() throws IOException { - if ((isNull(ids) || ids.isEmpty()) && isNull(filePath)) { - System.err.println( - "You have to provide one ID as command line argument or a file containing IDs."); - System.exit(1); - } else if (nonNull(ids) && nonNull(filePath)) { - System.err.println( - "Arguments --identifier and --file are mutually exclusive, please provide only one."); - System.exit(1); - } else if (nonNull(filePath)){ - ids = IdentifierParser.readProvidedIdentifiers(filePath.toFile()); - } - return ids; - } - -} diff --git a/src/main/java/life/qbic/io/parser/IdentifierParser.java b/src/main/java/life/qbic/io/parser/IdentifierParser.java deleted file mode 100644 index 33940ba..0000000 --- a/src/main/java/life/qbic/io/parser/IdentifierParser.java +++ /dev/null @@ -1,25 +0,0 @@ -package life.qbic.io.parser; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Scanner; - -public class IdentifierParser { - - /** - * Retrieve the identifiers from provided file - * - * @return Identifiers for which datasets will be retrieved - * @throws IOException if the file could not be read successfully - */ - public static List readProvidedIdentifiers(File file) throws IOException { - List identifiers = new ArrayList<>(); - Scanner scanner = new Scanner(file); - while (scanner.hasNext()) { - identifiers.add(scanner.nextLine()); - } - return identifiers; - } -} diff --git a/src/main/java/life/qbic/model/Configuration.java b/src/main/java/life/qbic/model/Configuration.java deleted file mode 100644 index cb567bd..0000000 --- a/src/main/java/life/qbic/model/Configuration.java +++ /dev/null @@ -1,13 +0,0 @@ -package life.qbic.model; - -import java.nio.file.Path; -import java.nio.file.Paths; - -/** - * Global configuration container - */ -public class Configuration { - - public static final long MAX_DOWNLOAD_ATTEMPTS = 3; - public static final Path LOG_PATH = Paths.get(System.getProperty("user.dir"),"logs"); -} diff --git a/src/main/java/life/qbic/model/download/Authentication.java b/src/main/java/life/qbic/model/download/Authentication.java deleted file mode 100644 index 5a17c30..0000000 --- a/src/main/java/life/qbic/model/download/Authentication.java +++ /dev/null @@ -1,61 +0,0 @@ -package life.qbic.model.download; - -import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi; -import ch.systemsx.cisd.common.spring.HttpInvokerUtils; - -public class Authentication { - - private static String sessionToken; - private String user; - private String password; - private final IApplicationServerApi applicationServer; - - - public Authentication( - String user, - String password, - String AppServerUri) { - this.user = user; - this.password = password; - if (!AppServerUri.isEmpty()) { - this.applicationServer = - HttpInvokerUtils.createServiceStub( - IApplicationServerApi.class, AppServerUri + IApplicationServerApi.SERVICE_URL, 10000); - } else { - this.applicationServer = null; - } - this.setCredentials(user, password); - } - - /** - * Login method for openBIS authentication - * - * @throws AuthenticationException in case the authentication failed - */ - public void login() throws ConnectionException, AuthenticationException { - try { - sessionToken = applicationServer.login(user, password); - } catch (Exception e) { - throw new ConnectionException("Connection to openBIS server failed.", e); - } - if (sessionToken == null || sessionToken.isEmpty()) { - throw new AuthenticationException("Authentication failed. Are you using the correct " - + "credentials for http://qbic.life?"); - } - } - - public String getSessionToken() { - return sessionToken; - } - - /** - * Setter for user and password credentials - * - * @param user The openBIS user - * @param password The openBIS user's password - */ - public void setCredentials(String user, String password) { - this.user = user; - this.password = password; - } -} diff --git a/src/main/java/life/qbic/model/download/AuthenticationException.java b/src/main/java/life/qbic/model/download/AuthenticationException.java deleted file mode 100644 index 6738853..0000000 --- a/src/main/java/life/qbic/model/download/AuthenticationException.java +++ /dev/null @@ -1,23 +0,0 @@ -package life.qbic.model.download; - -/** - * Exception to indicate failed authentication against openBIS. - *

- * This exception shall be thrown, when the returned session token of openBIS is empty, after the - * client tried to authenticate against the openBIS application server via its Java API. - */ -public class AuthenticationException extends RuntimeException { - - AuthenticationException() { - super(); - } - - AuthenticationException(String msg) { - super(msg); - } - - AuthenticationException(String msg, Throwable t) { - super(msg, t); - } - -} diff --git a/src/main/java/life/qbic/model/download/OutputPathFinder.java b/src/main/java/life/qbic/model/download/OutputPathFinder.java deleted file mode 100644 index 2dbb977..0000000 --- a/src/main/java/life/qbic/model/download/OutputPathFinder.java +++ /dev/null @@ -1,83 +0,0 @@ -package life.qbic.model.download; - -import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -/** - * Methods to determine the final path for the output directory. - * The requested data will be downloaded into this directory. - */ -public class OutputPathFinder { - - private static final Logger LOG = LogManager.getLogger(OutputPathFinder.class); - - /** - * @param path to be shortened - * @return path that has no parents (top directory) - */ - private static Path getTopDirectory(Path path) { - Path currentPath = Paths.get(path.toString()); - Path parentPath; - while (currentPath.getParent() != null) { - parentPath = currentPath.getParent(); - currentPath = parentPath; - } - return currentPath; - } - - /** - * @param possiblePath: string that could be an existing Path to a directory - * @return true if path exists, false otherwise - */ - private static boolean isPathValid(String possiblePath){ - Path path = Paths.get(possiblePath); - return Files.isDirectory(path); - } - - /** - * @param file to download - * @param conservePaths if true, directory structure will be conserved - * @return final path to file itself - */ - private static Path determineFinalPathFromDataset(DataSetFile file, Boolean conservePaths ) { - Path finalPath; - if (conservePaths) { - finalPath = Paths.get(file.getPath()); - // drop top parent directory name in the openBIS DSS (usually "/origin") - Path topDirectory = getTopDirectory(finalPath); - finalPath = topDirectory.relativize(finalPath); - } else { - finalPath = Paths.get(file.getPath()).getFileName(); - } - return finalPath; - } - - /** - * @param outputPath provided by user - * @param prefix sample code - * @param file to download - * @param conservePaths provided by user - * @return output directory path - */ - public static Path determineOutputDirectory(String outputPath, Path prefix, DataSetFile file, boolean conservePaths){ - Path filePath = determineFinalPathFromDataset(file, conservePaths); - String path = File.separator + prefix.toString() + File.separator + filePath.toString(); - Path finalPath = Paths.get(""); - if (outputPath != null && !outputPath.isEmpty()) { - if(isPathValid(outputPath)) { - finalPath = Paths.get(outputPath + path); - } else{ - LOG.error("The path you provided does not exist."); - System.exit(1); - } - } else { - finalPath = Paths.get(System.getProperty("user.dir") + path); - } - return finalPath; - } -} diff --git a/src/main/java/life/qbic/model/download/QbicDataDisplay.java b/src/main/java/life/qbic/model/download/QbicDataDisplay.java deleted file mode 100644 index 3042e30..0000000 --- a/src/main/java/life/qbic/model/download/QbicDataDisplay.java +++ /dev/null @@ -1,147 +0,0 @@ -package life.qbic.model.download; - -import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; -import ch.ethz.sis.openbis.generic.dssapi.v3.IDataStoreServerApi; -import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; -import ch.systemsx.cisd.common.spring.HttpInvokerUtils; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.util.Comparator; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import life.qbic.model.files.FileSize; -import life.qbic.model.files.FileSizeFormatter; - -/** - * Lists information about requested datasets and their files - */ -public class QbicDataDisplay { - - final String sessionToken; - - private final static DateTimeFormatter utcDateTimeFormatterIso8601 = new DateTimeFormatterBuilder() - .appendPattern("yyyy-MM-dd'T'hh:mm:ss") - .appendZoneId() - .toFormatter() - .withZone(ZoneOffset.UTC); - - private final QbicDataFinder qbicDataFinder; - - private final boolean printWithChecksums; - - /** - * Constructor for a QbicDataDisplay instance - * - * @param AppServerUri The openBIS application server URL (AS) - * @param dataServerUris The openBIS datastore server URLs (DSS) - * @param sessionToken The session token for the datastore & application servers - */ - public QbicDataDisplay( - String AppServerUri, - List dataServerUris, - String sessionToken, - boolean printWithChecksums) { - this.printWithChecksums = printWithChecksums; - this.sessionToken = sessionToken; - IApplicationServerApi applicationServer; - if (!AppServerUri.isEmpty()) { - applicationServer = - HttpInvokerUtils.createServiceStub( - IApplicationServerApi.class, AppServerUri + IApplicationServerApi.SERVICE_URL, - 10000); - } else { - applicationServer = null; - } - List dataStoreServerApis = dataServerUris.stream() - .filter(dataStoreServerUri -> !dataStoreServerUri.isEmpty()) - .map(dataStoreServerUri -> - HttpInvokerUtils.createStreamSupportingServiceStub( - IDataStoreServerApi.class, dataStoreServerUri + IDataStoreServerApi.SERVICE_URL, - 10000)) - .collect(Collectors.toList()); - qbicDataFinder = new QbicDataFinder(applicationServer, dataStoreServerApis, sessionToken); - } - - public void getInformation(List ids, List suffixes){ - for (String ident : ids) { - Map> foundDataSets = qbicDataFinder.findAllDatasetsRecursive(ident); - int datasetCount = foundDataSets - .values().stream() - .mapToInt(List::size) - .sum(); - System.out.printf("Number of datasets found for identifier %s : %s %n", ident, - datasetCount); - if (foundDataSets.size() > 0) { - printInformation(foundDataSets, suffixes); - } - } - } - - private void printInformation(Map> sampleDataSets, List suffixes) { - for (Map.Entry> entry : sampleDataSets.entrySet()) { - for (DataSet dataSet : entry.getValue()) { - Predicate fileFilter = it -> true; - if (Objects.nonNull(suffixes) && !suffixes.isEmpty()) { - Predicate suffixFilter = file -> { - String fileName = getFileName(file); - return suffixes.stream() - .map(String::trim).map(String::toLowerCase) - .anyMatch(fileName.toLowerCase()::endsWith); - }; - fileFilter = fileFilter.and(suffixFilter); - } - List dataSetFiles = qbicDataFinder.getFiles(dataSet.getPermId(), fileFilter); - - //skip if no files found - if (dataSetFiles.isEmpty()) { - continue; - } - - long totalSize = dataSetFiles.stream().mapToLong(DataSetFile::getFileLength).sum(); - - Sample analyte = qbicDataFinder.searchAnalyteParent(entry.getKey()); - Date registrationDate = dataSet.getRegistrationDate(); - String iso_registrationDate = utcDateTimeFormatterIso8601.format(registrationDate.toInstant()); - int columnWidth = 16; - System.out.printf("# %-" + columnWidth + "s %s (%s)%n", "Dataset", - dataSet.getSample().getCode(), - dataSet.getPermId()); - System.out.printf("# %-" + columnWidth + "s %s%n", "Source", analyte.getCode()); - System.out.printf("# %-" + columnWidth + "s %s%n", "Registration", - iso_registrationDate); - System.out.printf("# %-" + columnWidth + "s %s%n", "Size", - FileSizeFormatter.format(FileSize.of(totalSize))); - - List sortedFiles = dataSetFiles.stream() - .sorted(Comparator.comparing(QbicDataDisplay::getFileName)) - .collect(Collectors.toList()); - - for (DataSetFile file : sortedFiles) { - String name = getFileName(file); - String fileSize = FileSizeFormatter.format(FileSize.of(file.getFileLength()),6); - int crc32 = file.getChecksumCRC32(); - if (printWithChecksums) { - System.out.printf("%s\t%08x\t%s%n", fileSize, crc32, name); - } else { - System.out.printf("%s\t%s%n", fileSize, name); - } - } - System.out.print("\n"); - } - } - } - - private static String getFileName(DataSetFile file) { - String filePath = file.getPermId().getFilePath(); - return filePath.substring(filePath.lastIndexOf("/") + 1); - } - - -} diff --git a/src/main/java/life/qbic/model/download/QbicDataDownloader.java b/src/main/java/life/qbic/model/download/QbicDataDownloader.java deleted file mode 100644 index 2d7e26f..0000000 --- a/src/main/java/life/qbic/model/download/QbicDataDownloader.java +++ /dev/null @@ -1,519 +0,0 @@ -package life.qbic.model.download; - -import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; -import ch.ethz.sis.openbis.generic.dssapi.v3.IDataStoreServerApi; -import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; -import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownload; -import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownloadOptions; -import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownloadReader; -import ch.systemsx.cisd.common.spring.HttpInvokerUtils; -import life.qbic.ChecksumReporter; -import life.qbic.DownloadException; -import life.qbic.DownloadRequest; -import life.qbic.FileSystemWriter; -import life.qbic.model.Configuration; -import life.qbic.util.ProgressBar; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.*; -import java.net.MalformedURLException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; -import java.util.Map.Entry; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.zip.CRC32; -import java.util.zip.CheckedInputStream; - -public class QbicDataDownloader { - - private static final Logger LOG = LogManager.getLogger(QbicDataDownloader.class); - private final int defaultBufferSize; - private final boolean conservePaths; - private final List dataStoreServers; - private final String sessionToken; - private final String outputPath; - - private final ChecksumReporter checksumReporter = new FileSystemWriter( - Paths.get(Configuration.LOG_PATH.toString(), "summary_valid_files.txt"), - Paths.get(Configuration.LOG_PATH.toString(), "summary_invalid_files.txt")); - private final QbicDataFinder qbicDataFinder; - - /** - * Constructor for a QBiCDataDownloader instance - * - * @param AppServerUri The openBIS application server URL (AS) - * @param dataServerUris The openBIS datastore server URLs (DSS) - * @param bufferSize The buffer size for the InputStream reader - * @param conservePaths Flag to conserve the file path structure during download - * @param sessionToken The session token for the datastore & application servers - */ - public QbicDataDownloader( - String AppServerUri, - List dataServerUris, - int bufferSize, - boolean conservePaths, - String sessionToken, - String outputPath) { - this.defaultBufferSize = bufferSize; - this.conservePaths = conservePaths; - this.sessionToken = sessionToken; - this.outputPath = outputPath; - - IApplicationServerApi applicationServer; - if (!AppServerUri.isEmpty()) { - applicationServer = - HttpInvokerUtils.createServiceStub( - IApplicationServerApi.class, AppServerUri + IApplicationServerApi.SERVICE_URL, 10000); - } else { - applicationServer = null; - } - this.dataStoreServers = dataServerUris.stream() - .filter(dataStoreServerUri -> !dataStoreServerUri.isEmpty()) - .map(dataStoreServerUri -> - HttpInvokerUtils.createStreamSupportingServiceStub( - IDataStoreServerApi.class, dataStoreServerUri + IDataStoreServerApi.SERVICE_URL, - 10000)) - .collect(Collectors.toList()); - qbicDataFinder = new QbicDataFinder(applicationServer, dataStoreServers, sessionToken); - } - - /** - * Filters the files as specified by the user and downloads them. - */ - public DownloadResponse downloadForIds(List ids, List suffixes) { - LOG.info( - String.format( - "%s provided openBIS identifiers have been found: %s", - ids.size(), ids)); - Predicate fileFilter = it -> true; - if (Objects.nonNull(suffixes) && !suffixes.isEmpty()) { - LOG.info( - "Applying suffix filter for suffixes: [" + String.join(" ", suffixes) + "] ..."); - Predicate suffixFilter = file -> { - String fileName = getFileName(file); - return suffixes.stream().anyMatch(fileName::endsWith); - }; - fileFilter = fileFilter.and(suffixFilter); - } - - DownloadResponse downloadResponse = DownloadResponse.create(); - for (String ident : ids) { - downloadResponse.merge(downloadForId(fileFilter, ident)); - } - return downloadResponse; - } - - private DownloadResponse downloadForId(Predicate fileFilter, String ident) { - Map> foundDataSets = qbicDataFinder.findAllDatasetsRecursive(ident); - - if (foundDataSets.size() > 0) { - Map> datasetsPerAnalyte = associateDatasetsWithAnalyte(foundDataSets); - - // ensures the order of download is the same always - List>> sortedEntries = datasetsPerAnalyte - .entrySet().stream() - .sorted(Comparator.comparing(entry -> entry.getKey().getCode())) - .collect(Collectors.toList()); - - DownloadResponse response = DownloadResponse.create(); - for (Entry> analyteDatasets : sortedEntries) { - response.merge(downloadForAnalyte(fileFilter, analyteDatasets)); - } - return response; - } else { - LOG.info("Nothing to download for " + ident + "."); - return DownloadResponse.create(); - } - } - - private DownloadResponse downloadForAnalyte(Predicate fileFilter, - Entry> analyteDatasets) { - - Sample analyte = analyteDatasets.getKey(); - List dataSets = analyteDatasets.getValue(); - return downloadDatasets(analyte, dataSets, fileFilter); - } - - private Map> associateDatasetsWithAnalyte(Map> foundDataSets) { - Map> datasetsPerAnalyte = new HashMap<>(); - for (Entry> sampleListEntry : foundDataSets.entrySet()) { - Sample analyte = qbicDataFinder.searchAnalyteParent(sampleListEntry.getKey()); - datasetsPerAnalyte.putIfAbsent(analyte, new ArrayList<>()); - datasetsPerAnalyte.get(analyte).addAll(sampleListEntry.getValue()); - } - return datasetsPerAnalyte; - } - - private String getFileName(DataSetFile file) { - String filePath = file.getPermId().getFilePath(); - return filePath.substring(filePath.lastIndexOf("/") + 1); - } - - private DownloadResponse downloadDatasets(Sample analyte, List dataSets, - Predicate fileFilter) { - - if (dataSets.isEmpty()) { - return DownloadResponse.create(); - } - - List sortedDatasets = dataSets.stream() - .sorted(Comparator.comparing(it -> it.getSample().getCode())) - .collect(Collectors.toList()); - - LOG.info(String.format("Found " + sortedDatasets.size() + " dataset(s) for sample %s", - analyte.getCode())); - DownloadResponse downloadResponse = DownloadResponse.create(); - for (DataSet dataSet : sortedDatasets) { - DataSetPermId permID = dataSet.getPermId(); - List files = qbicDataFinder.getFiles(permID, fileFilter); - String datasetSample = dataSet.getSample().getCode(); - if (files.isEmpty()) { - LOG.info("Nothing to download for dataset " + datasetSample + " (" + permID + ")" + "."); - continue; - } - final DownloadRequest downloadRequest = new DownloadRequest( - Paths.get(datasetSample + File.separator), files, Configuration.MAX_DOWNLOAD_ATTEMPTS); - LOG.info( - "Downloading " + files.size() + " file" + (files.size() != 1 ? "s" : "") + " for dataset " - + datasetSample + " (" + permID + ")"); - downloadResponse.merge(downloadFiles(downloadRequest)); - } - return downloadResponse; - } - - private FileDownloadResponse downloadFile(DataSetFile dataSetFile, Path prefix) throws DownloadException { - for (IDataStoreServerApi dataStoreServer : dataStoreServers) { - try { - return downloadFileFromDataStoreServer(dataSetFile, prefix, dataStoreServer); - } catch (FileNotFoundException fileNotFoundException) { - // log and try the next data store server - LOG.debug("Download attempt failed on " + dataStoreServer, fileNotFoundException); - } - } - return FileDownloadResponse.failure(); - } - - // note: DataSetFileDownloadReader closes the input stream after it finished reading it. - private static class AutoClosableDataSetFileDownloadReader extends DataSetFileDownloadReader implements AutoCloseable { - public AutoClosableDataSetFileDownloadReader(InputStream in) { - super(in); - } - } - - public static class FileDownloadResponse { - private final boolean success; - - private FileDownloadResponse(boolean success) { - this.success = success; - } - - public static FileDownloadResponse success() { - return new FileDownloadResponse(true); - } - - private static FileDownloadResponse failure() { - return new FileDownloadResponse(false); - } - - public boolean isSuccess() { - return success; - } - - public boolean isFailure() { - return !success; - } - } - - - /** - * @throws FileNotFoundException if the file was not on the data store server. - */ - private FileDownloadResponse downloadFileFromDataStoreServer(DataSetFile dataSetFile, Path prefix, - IDataStoreServerApi dataStoreServerApi) throws FileNotFoundException { - - // skip if file is empty - if (dataSetFile.getFileLength() < 1) { - LOG.info("Skipped empty file " + dataSetFile.getPath()); - return FileDownloadResponse.success(); - } - - Path localFilePath = OutputPathFinder.determineOutputDirectory(outputPath, prefix, dataSetFile, - conservePaths); - File localFile = localFilePath.toFile(); - - // skip if same content exists already - if (localFile.exists()) { - System.out.print("Found existing file. Checking content..."); - - if (localFileWithSameContent(dataSetFile, localFile)) { - System.out.println("\rFound existing file with identical content. Skipping " - + localFile.getAbsolutePath()); - return FileDownloadResponse.success(); - } else { - System.out.println("\rUpdating existing file " + localFile.getAbsolutePath()); - } - } - - createParentDirectoryIfNotExists(localFile); - - long computedChecksum = writeFileToDisk(dataSetFile, dataStoreServerApi, localFilePath); - // validate written file - ChecksumValidationResult checksumValidationResult = validateChecksum(computedChecksum, - dataSetFile); - reportValidation(checksumValidationResult); - if (checksumValidationResult.isValid()) { - LOG.info("Download successful for " + localFile.getAbsolutePath()); - return FileDownloadResponse.success(); - } else { - LOG.warn(String.format( - "Checksum mismatches were detected for file %s.%nFor more Information check the logs/summary_invalid_files.txt log file.", - localFile.getAbsolutePath())); - return FileDownloadResponse.failure(); - } - } - - private long writeFileToDisk(DataSetFile dataSetFile, IDataStoreServerApi dataStoreServerApi, - Path localFilePath) throws FileNotFoundException { - - long computedChecksum; - DataSetFileDownloadOptions options = new DataSetFileDownloadOptions(); - options.setRecursive(false); - try (AutoClosableDataSetFileDownloadReader reader = new AutoClosableDataSetFileDownloadReader( - dataStoreServerApi.downloadFiles(sessionToken, - Collections.singletonList(dataSetFile.getPermId()), options))) { - - DataSetFileDownload fileDownload = reader.read(); - // there is no file in the input stream - if (Objects.isNull(fileDownload)) { - throw new FileNotFoundException("No file " + dataSetFile.getPermId() + " found"); - } - - // write the file - try ( - InputStream initialStream = fileDownload.getInputStream(); - OutputStream os = Files.newOutputStream(localFilePath); - CheckedInputStream checkedInputStream = new CheckedInputStream(initialStream, - new CRC32())) { - - long fileLength = dataSetFile.getFileLength(); - String fileName = localFilePath.getFileName().toString(); - ProgressBar progressBar = new ProgressBar(fileName, fileLength); - int bufferSize = adjustedBufferSize(fileLength); - - byte[] buffer = new byte[bufferSize]; - int bytesRead; - while ((bytesRead = checkedInputStream.read(buffer)) != -1) { - progressBar.updateProgress(bufferSize); - os.write(buffer, 0, bytesRead); - os.flush(); - } - progressBar.remove(); - // flush OutputStream to write any buffered data to file - os.flush(); - - computedChecksum = checkedInputStream.getChecksum().getValue(); - } catch (IOException e) { - throw new DownloadException(e); - } - } - return computedChecksum; - } - - private boolean localFileWithSameContent(DataSetFile dataSetFile, File localFile) { - ChecksumValidationResult checksumValidationResult = checksumForFile(dataSetFile, localFile); - if (checksumValidationResult.isValid()) { - LOG.debug(checksumValidationResult.computedChecksum + " " + localFile.getName() - + " exists locally."); - return true; - } - return false; - } - - private static void createParentDirectoryIfNotExists(File localFile) { - if (!localFile.getParentFile().exists()) { - boolean successfullyCreatedDirectory = localFile.getParentFile().mkdirs(); - if (!successfullyCreatedDirectory) { - throw new DownloadException("Could not create directory " + localFile.getParentFile()); - } - } - } - - private ChecksumValidationResult checksumForFile(DataSetFile dataSetFile, File localFile) { - try (CheckedInputStream existingFileReader = new CheckedInputStream( - Files.newInputStream(localFile.toPath()), new CRC32())) { - int bufferSize = adjustedBufferSize(dataSetFile.getFileLength()); - byte[] buffer = new byte[bufferSize]; - while (existingFileReader.read(buffer) != -1) { - // reading - } - return validateChecksum( - existingFileReader.getChecksum().getValue(), dataSetFile); - } catch (IOException e) { - throw new RuntimeException("Existing file could not be processed", e); - } - } - - private int adjustedBufferSize(long fileLength) { - return (fileLength < defaultBufferSize) ? (int) fileLength : defaultBufferSize; - } - - private static class ChecksumValidationResult { - private final String expectedChecksum; - private final String computedChecksum; - private final DataSetFile dataSetFile; - - - private ChecksumValidationResult(String expectedChecksum, String computedChecksum, - DataSetFile dataSetFile) { - Objects.requireNonNull(expectedChecksum); - Objects.requireNonNull(computedChecksum); - Objects.requireNonNull(dataSetFile); - - this.expectedChecksum = expectedChecksum; - this.computedChecksum = computedChecksum; - this.dataSetFile = dataSetFile; - } - - public String expectedChecksum() { - return expectedChecksum; - } - - public String computedChecksum() { - return computedChecksum; - } - - public DataSetFile dataSetFile() { - return dataSetFile; - } - - public boolean isValid() { - return computedChecksum.equals(expectedChecksum); - } - - public boolean isInvalid() { - return !isValid(); - } - } - - - private void reportValidation(ChecksumValidationResult validation) { - try { - if (validation.isValid()) { - checksumReporter.reportMatchingChecksum( - validation.expectedChecksum(), - validation.computedChecksum(), - Paths.get(validation.dataSetFile().getPath()).toUri().toURL()); - } else { - checksumReporter.reportMismatchingChecksum( - validation.expectedChecksum(), - validation.computedChecksum(), - Paths.get(validation.dataSetFile().getPath()).toUri().toURL()); - } - } catch (MalformedURLException e) { - LOG.error(e); - } - } - - private ChecksumValidationResult validateChecksum(long computedChecksum, DataSetFile dataSetFile) { - String expectedChecksumHex = Integer.toHexString(dataSetFile.getChecksumCRC32()); - String computedChecksumHex = Long.toHexString(computedChecksum); - return new ChecksumValidationResult( - expectedChecksumHex, computedChecksumHex, dataSetFile); - } - - public static class DownloadResponse { - - private final List responses; - - private DownloadResponse() { - responses = new ArrayList<>(); - } - - public static DownloadResponse create() { - return new DownloadResponse(); - } - - public void add(FileDownloadResponse response) { - responses.add(response); - } - - public DownloadResponse merge(DownloadResponse response) { - responses.addAll(response.responses); - return this; - } - - public boolean containsFailure() { - return responses.stream().anyMatch(FileDownloadResponse::isFailure); - } - - public long failureCount() { - return responses.stream().filter(FileDownloadResponse::isFailure).count(); - } - - public long fileCount() { - return responses.size(); - } - - } - private DownloadResponse downloadFiles(DownloadRequest request) throws DownloadException { - List dataSetFiles = request.getFiles().stream() - .sorted(Comparator.comparing(this::getFileName)) - .collect(Collectors.toList()); - DownloadResponse downloadResponse = DownloadResponse.create(); - for (DataSetFile dataSetFile : dataSetFiles) { - FileDownloadResponse fileDownloadResponse = attemptFileDownload(request.getPrefix(), - dataSetFile, request.getMaxNumberOfAttempts()); - downloadResponse.add(fileDownloadResponse); - } - - assert dataSetFiles.size() == downloadResponse.fileCount() : "each file gave a response"; - - if (downloadResponse.containsFailure()) { - LOG.warn(String.format("Failed to download %s out of %s files", - downloadResponse.failureCount(), dataSetFiles.size())); - } - return downloadResponse; - } - - private FileDownloadResponse attemptFileDownload(Path pathPrefix, - DataSetFile dataSetFile, long maxNumberOfAttempts) throws DownloadException { - int downloadAttempt = 1; - assert maxNumberOfAttempts >= 1 : "max download attempts are at least 1"; - while (true) { - if (downloadAttempt > maxNumberOfAttempts) { - LOG.error("Maximum number of download attempts reached."); - return FileDownloadResponse.failure(); - } - try { - FileDownloadResponse fileDownloadResponse = downloadFile(dataSetFile, pathPrefix); - writeCRC32Checksum(dataSetFile, pathPrefix); - if (fileDownloadResponse.isFailure()) { - LOG.error("Download attempt " + downloadAttempt + " failed."); - downloadAttempt++; - continue; - } - return fileDownloadResponse; - } catch (Exception e) { - LOG.error(String.format("Download attempt %d failed.", downloadAttempt)); - LOG.error(String.format("Reason: %s", e.getMessage())); - LOG.debug(e); - downloadAttempt++; - } - } - } - - private void writeCRC32Checksum(DataSetFile dataSetFile, Path pathPrefix) { - - Path path = OutputPathFinder.determineOutputDirectory(outputPath, pathPrefix ,dataSetFile, conservePaths); - - checksumReporter.storeChecksum(path, Integer.toHexString(dataSetFile.getChecksumCRC32())); - } - -} diff --git a/src/main/java/life/qbic/model/download/QbicDataFinder.java b/src/main/java/life/qbic/model/download/QbicDataFinder.java deleted file mode 100644 index b0c22b3..0000000 --- a/src/main/java/life/qbic/model/download/QbicDataFinder.java +++ /dev/null @@ -1,136 +0,0 @@ -package life.qbic.model.download; - -import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.search.SearchResult; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.fetchoptions.DataSetFetchOptions; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleFetchOptions; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.search.SampleSearchCriteria; -import ch.ethz.sis.openbis.generic.dssapi.v3.IDataStoreServerApi; -import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; -import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.fetchoptions.DataSetFileFetchOptions; -import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.search.DataSetFileSearchCriteria; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class QbicDataFinder { - - private static final Logger log = LogManager.getLogger(QbicDataFinder.class); - - private final IApplicationServerApi applicationServer; - - private final List dataStoreServers; - - private final String sessionToken; - - public QbicDataFinder( - IApplicationServerApi applicationServer, - List dataStoreServers, - String sessionToken) { - this.applicationServer = applicationServer; - this.dataStoreServers = dataStoreServers; - this.sessionToken = sessionToken; - } - - /** - * Fetches all datasets, even those of children - recursively - * - * @param sample the sample for which descending data sets should be added - * @param visitedSamples map with samples and datasets already visited. - */ - private static void fillWithDescendantDatasets(Sample sample, - Map> visitedSamples) { - if (visitedSamples.containsKey(sample)) { - return; - } - - List children = sample.getChildren(); - List foundDataSets = sample.getDataSets(); - visitedSamples.put(sample, foundDataSets); - // recursion end - if (children.size() > 0) { - for (Sample child : children) { - fillWithDescendantDatasets(child, visitedSamples); - } - } - } - - public List getFiles(DataSetPermId permID, Predicate fileFilter) { - DataSetFileSearchCriteria criteria = new DataSetFileSearchCriteria(); - criteria.withDataSet().withCode().thatEquals(permID.getPermId()); - List files = new ArrayList<>(); - // add files from all data store servers - for (IDataStoreServerApi dataStoreServer : dataStoreServers) { - List filesOnDataStoreServer = dataStoreServer - .searchFiles(sessionToken, criteria, new DataSetFileFetchOptions()) - .getObjects(); - if (filesOnDataStoreServer.isEmpty()) { - log.debug( - String.format("No files found in dataset %s on dss %s", permID, dataStoreServer)); - } else { - log.debug(String.format("%s files and directories found in dataset %s on dss %s", - filesOnDataStoreServer.size(), permID, dataStoreServer)); - } - files.addAll(filesOnDataStoreServer); - } - - Predicate notADirectory = dataSetFile -> !dataSetFile.isDirectory(); - return files.stream().filter(notADirectory.and(fileFilter)).collect(Collectors.toList()); - } - - /** - * Finds all datasets of a given sampleID, even those of its children - recursively - * - * @param sampleId provided by user - * @return all found datasets for a given sampleID - */ - public Map> findAllDatasetsRecursive(String sampleId) { - Map> dataSetsBySample = new HashMap<>(); - - SampleSearchCriteria criteria = new SampleSearchCriteria(); - criteria.withCode().thatEquals(sampleId); - - // tell the API to fetch all descendants for each returned sample - SampleFetchOptions fetchOptions = new SampleFetchOptions(); - DataSetFetchOptions dsFetchOptions = new DataSetFetchOptions(); - dsFetchOptions.withType(); - dsFetchOptions.withSample(); - fetchOptions.withType(); - fetchOptions.withChildrenUsing(fetchOptions); - fetchOptions.withParentsUsing(fetchOptions); - fetchOptions.withDataSetsUsing(dsFetchOptions); - - SearchResult result = - applicationServer.searchSamples(sessionToken, criteria, fetchOptions); - List samples = result.getObjects(); - - for (Sample sample : samples) { - fillWithDescendantDatasets(sample, dataSetsBySample); - } - return dataSetsBySample; - } - - /** - * Searches the parents for a Q_TEST_SAMPLE assuming at most one Q_TEST_SAMPLE exists in the parent - * samples. If no Q_TEST_SAMPLE was found, the original sample is returned. - * - * @param sample the sample to which a dataset is attached to - * @return the Q_TEST_SAMPLE parent if exists, the sample itself otherwise. - */ - public Sample searchAnalyteParent(Sample sample) { - Optional firstTestSample = sample.getParents().stream() - .filter( - it -> it.getType().getCode().equals("Q_TEST_SAMPLE")) - .findFirst(); - return firstTestSample.orElse(sample); - } -} diff --git a/src/main/java/life/qbic/qpostman/common/AuthenticationException.java b/src/main/java/life/qbic/qpostman/common/AuthenticationException.java new file mode 100644 index 0000000..b16b645 --- /dev/null +++ b/src/main/java/life/qbic/qpostman/common/AuthenticationException.java @@ -0,0 +1,40 @@ +package life.qbic.qpostman.common; + +/** + * Exception to indicate failed authentication against openBIS. + *

+ * This exception shall be thrown, when the returned session token of openBIS is empty, after the + * client tried to authenticate against the openBIS application server via its Java API. + */ +public class AuthenticationException extends RuntimeException { + + private final String username; + + public AuthenticationException(String username) { + this.username = username; + } + + public AuthenticationException(String message, String username) { + super(message); + this.username = username; + } + + public AuthenticationException(String message, Throwable cause, String username) { + super(message, cause); + this.username = username; + } + + public AuthenticationException(Throwable cause, String username) { + super(cause); + this.username = username; + } + + public AuthenticationException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace, String username) { + super(message, cause, enableSuppression, writableStackTrace); + this.username = username; + } + + public String username() { + return username; + } +} diff --git a/src/main/java/life/qbic/model/files/FileSizeFormatter.java b/src/main/java/life/qbic/qpostman/common/FileSizeFormatter.java similarity index 96% rename from src/main/java/life/qbic/model/files/FileSizeFormatter.java rename to src/main/java/life/qbic/qpostman/common/FileSizeFormatter.java index 2a5eb20..59acaf5 100644 --- a/src/main/java/life/qbic/model/files/FileSizeFormatter.java +++ b/src/main/java/life/qbic/qpostman/common/FileSizeFormatter.java @@ -1,10 +1,11 @@ -package life.qbic.model.files; +package life.qbic.qpostman.common; import java.math.BigDecimal; import java.math.RoundingMode; import java.text.DecimalFormat; import java.util.Arrays; import java.util.Comparator; +import life.qbic.qpostman.common.structures.FileSize; /** * Formats a file size to human-readable form. @@ -80,6 +81,4 @@ public static String format(FileSize fileSize) { return format(fileSize, 1); } - - } diff --git a/src/main/java/life/qbic/io/commandline/ManifestVersionProvider.java b/src/main/java/life/qbic/qpostman/common/ManifestVersionProvider.java similarity index 93% rename from src/main/java/life/qbic/io/commandline/ManifestVersionProvider.java rename to src/main/java/life/qbic/qpostman/common/ManifestVersionProvider.java index afa48a3..db304a1 100644 --- a/src/main/java/life/qbic/io/commandline/ManifestVersionProvider.java +++ b/src/main/java/life/qbic/qpostman/common/ManifestVersionProvider.java @@ -1,4 +1,4 @@ -package life.qbic.io.commandline; +package life.qbic.qpostman.common; import picocli.CommandLine; diff --git a/src/main/java/life/qbic/qpostman/common/PostmanCommand.java b/src/main/java/life/qbic/qpostman/common/PostmanCommand.java new file mode 100644 index 0000000..7692e61 --- /dev/null +++ b/src/main/java/life/qbic/qpostman/common/PostmanCommand.java @@ -0,0 +1,25 @@ +package life.qbic.qpostman.common; + +import static picocli.CommandLine.Command; + +import life.qbic.qpostman.download.DownloadCommand; +import life.qbic.qpostman.list.ListCommand; +import picocli.CommandLine; + +@Command(name = "postman-cli", + descriptionHeading = "%nDescription:%n", + description = "A software client for downloading data from QBiC.", + parameterListHeading = "%nParameters:%n", + optionListHeading = "%nOptions:%n", + commandListHeading = "%nCommands:%n", + synopsisSubcommandLabel = "COMMAND", + footer = "Optional: specify a config file by running postman with '@/path/to/config.txt'.\nA detailed documentation can be found at \nhttps://github.com/qbicsoftware/postman-cli#readme.", + footerHeading = "%n", + mixinStandardHelpOptions = true, + scope = CommandLine.ScopeType.INHERIT, + sortOptions = false, + usageHelpAutoWidth = true, + versionProvider = ManifestVersionProvider.class, + subcommands = {DownloadCommand.class, ListCommand.class}) +public class PostmanCommand { +} diff --git a/src/main/java/life/qbic/util/ProgressBar.java b/src/main/java/life/qbic/qpostman/common/ProgressBar.java similarity index 83% rename from src/main/java/life/qbic/util/ProgressBar.java rename to src/main/java/life/qbic/qpostman/common/ProgressBar.java index 74ea198..6c27092 100644 --- a/src/main/java/life/qbic/util/ProgressBar.java +++ b/src/main/java/life/qbic/qpostman/common/ProgressBar.java @@ -1,19 +1,17 @@ -package life.qbic.util; - -import jline.TerminalFactory; -import life.qbic.model.files.FileSize; -import life.qbic.model.files.FileSizeFormatter; +package life.qbic.qpostman.common; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Date; import java.util.TimeZone; +import life.qbic.qpostman.common.structures.FileSize; public class ProgressBar { - private final int BARSIZE = TerminalFactory.get().getWidth() / 3; - private final int MAXFILENAMESIZE = TerminalFactory.get().getWidth() / 3; + private static final int TERMINAL_WIDTH = 80; + private final int BARSIZE = TERMINAL_WIDTH / 3; + private final int MAXFILENAMESIZE = TERMINAL_WIDTH / 3; private static final long UPDATE_INTERVAL = 1000; private float nextProgressJump; private final float stepSize; @@ -33,8 +31,8 @@ public ProgressBar(String fileName, long totalFileSize) { lastUpdated = 0; } - public void updateProgress(int addDownloadedSize) { - this.downloadedSize += (long) addDownloadedSize; + public void updateProgress(long addDownloadedSize) { + this.downloadedSize += addDownloadedSize; update(); } @@ -77,7 +75,7 @@ private void drawProgress() { } public void remove() { - System.out.printf("\r%"+TerminalFactory.get().getWidth()+"s\r", ""); //clear whole line + System.out.printf("\r%"+TERMINAL_WIDTH+"s\r", ""); //clear whole line } private int computeLeftPadding() { @@ -99,12 +97,8 @@ private String buildProgressBar() { dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); String remainingTime = dateFormat.format(new Date(remainingDownloadTime)); - for (int i = 0; i < numberProgressStrings; i++) { - progressBar.append("#"); - } - for (int i = numberProgressStrings; i < BARSIZE; i++) { - progressBar.append(" "); - } + progressBar.append("#".repeat(Math.max(0, numberProgressStrings))); + progressBar.append(" ".repeat(Math.max(0, BARSIZE - numberProgressStrings))); progressBar.append("]\t"); FileSize downloadedSize = FileSize.of(this.downloadedSize); diff --git a/src/main/java/life/qbic/qpostman/common/functions/EndsWithIgnoreCase.java b/src/main/java/life/qbic/qpostman/common/functions/EndsWithIgnoreCase.java new file mode 100644 index 0000000..9d3e296 --- /dev/null +++ b/src/main/java/life/qbic/qpostman/common/functions/EndsWithIgnoreCase.java @@ -0,0 +1,16 @@ +package life.qbic.qpostman.common.functions; + +import java.util.function.BiFunction; + +public class EndsWithIgnoreCase implements BiFunction { + + public static boolean endsWithIgnoreCase(String input, String suffix) { + int suffixLength = suffix.length(); + return input.regionMatches(true, input.length() - suffixLength, suffix, 0, suffixLength); + } + + @Override + public Boolean apply(String input, String suffix) { + return endsWithIgnoreCase(input, suffix); + } +} diff --git a/src/main/java/life/qbic/qpostman/common/functions/FileFilter.java b/src/main/java/life/qbic/qpostman/common/functions/FileFilter.java new file mode 100644 index 0000000..f6b86ba --- /dev/null +++ b/src/main/java/life/qbic/qpostman/common/functions/FileFilter.java @@ -0,0 +1,44 @@ +package life.qbic.qpostman.common.functions; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Predicate; +import life.qbic.qpostman.common.structures.DataFile; + +/** + * A class implementing the {@link Predicate} interface and providing file filtering functionality based on suffixes. + */ +public class FileFilter implements Predicate { + + private final List suffixes; + private final boolean caseSensitive; + + public static FileFilter create() { + return new FileFilter(new ArrayList<>(), false); + } + + public FileFilter withSuffixes(List suffixes) { + var temp = new ArrayList<>(this.suffixes); + temp.addAll(suffixes); + return new FileFilter(temp, caseSensitive); + } + + private FileFilter(List suffixes, boolean caseSensitive) { + this.suffixes = suffixes; + this.caseSensitive = caseSensitive; + } + + @Override + public boolean test(DataFile dataFile) { + boolean result = true; + if (!suffixes.isEmpty()) { + result &= suffixes.stream() + .anyMatch(suffix -> hasSuffix(dataFile.fileName(), suffix)); + } + return result; + } + + private static boolean hasSuffix(String input, String suffix) { + return input.toLowerCase().endsWith(suffix.toLowerCase()); + } +} diff --git a/src/main/java/life/qbic/qpostman/common/functions/FindSourceSample.java b/src/main/java/life/qbic/qpostman/common/functions/FindSourceSample.java new file mode 100644 index 0000000..18232bc --- /dev/null +++ b/src/main/java/life/qbic/qpostman/common/functions/FindSourceSample.java @@ -0,0 +1,37 @@ +package life.qbic.qpostman.common.functions; + +import static java.util.Objects.requireNonNull; + +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; +import java.util.Optional; +import java.util.function.Function; + +public class FindSourceSample implements Function { + private final String sourceSampleTypeCode; + + public FindSourceSample(String sourceSampleTypeCode) { + this.sourceSampleTypeCode = sourceSampleTypeCode; + } + + @Override + public Sample apply(Sample sample) { + requireNonNull(sample, "sample must not be null"); + return findSourceSample(sample).orElse(sample); + } + + private Optional findSourceSample(Sample sample) { + if (sample.getType().getCode().equals(sourceSampleTypeCode)) { + return Optional.of(sample); + } + if (sample.getParents().isEmpty()) { + return Optional.empty(); + } + for (Sample parent : sample.getParents()) { + Optional sourceSample = findSourceSample(parent); + if (sourceSample.isPresent()) { + return sourceSample; + } + } + return Optional.empty(); + } +} diff --git a/src/main/java/life/qbic/qpostman/common/functions/SearchDataSets.java b/src/main/java/life/qbic/qpostman/common/functions/SearchDataSets.java new file mode 100644 index 0000000..caae664 --- /dev/null +++ b/src/main/java/life/qbic/qpostman/common/functions/SearchDataSets.java @@ -0,0 +1,93 @@ +package life.qbic.qpostman.common.functions; + +import static java.util.Objects.requireNonNull; + +import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleFetchOptions; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.search.SampleSearchCriteria; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Stream; +import life.qbic.qpostman.common.structures.DataSetWrapper; +import life.qbic.qpostman.openbis.OpenBisSessionProvider; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class SearchDataSets implements Function, Collection> { + private static final Logger log = LogManager.getLogger(SearchDataSets.class); + private final IApplicationServerApi applicationServerApi; + + public SearchDataSets(IApplicationServerApi applicationServerApi) { + this.applicationServerApi = applicationServerApi; + } + + private Collection searchDataSets(Collection userInput) { + List samples = userInput.stream() + .map(SampleQuery::new) + .flatMap(this::searchSamples) + .toList(); + Set processedSampleCodes = new HashSet<>(); + Set foundDataSets = new HashSet<>(); + for (Sample it : samples) { + addAllDataSets(it, processedSampleCodes, foundDataSets); + } + return foundDataSets; + } + + private Stream searchSamples(SampleQuery sampleQuery) { + return applicationServerApi.searchSamples(OpenBisSessionProvider.get().getToken(), sampleQuery.searchCriteria(), sampleQuery.fetchOptions()).getObjects().stream(); + } + + @Override + public Collection apply(Collection strings) { + return searchDataSets(strings); + } + + private record SampleQuery(String sampleCode) { + SampleQuery { + requireNonNull(sampleCode, "sampleCode must not be null"); + } + public SampleSearchCriteria searchCriteria() { + SampleSearchCriteria sampleSearchCriteria = new SampleSearchCriteria(); + sampleSearchCriteria.withCode().thatEquals(sampleCode); + return sampleSearchCriteria; + } + + public SampleFetchOptions fetchOptions() { + SampleFetchOptions parentFetchOptions = new SampleFetchOptions(); + parentFetchOptions.withType(); + SampleFetchOptions sampleFetchOptions = new SampleFetchOptions(); + sampleFetchOptions.withDataSets().withSample(); + sampleFetchOptions.withType(); + sampleFetchOptions.withParents(); + sampleFetchOptions.withParentsUsing(parentFetchOptions); + sampleFetchOptions.withChildrenUsing(sampleFetchOptions); + return sampleFetchOptions; + } + } + + private void addAllDataSets(Sample sample, Set processedSampleCodes, Set dataSetAccumulator) { + if (processedSampleCodes.contains(sample.getCode())) { + log.trace("already visited " + sample.getCode()); + return; + } else { + log.trace("visiting " + sample.getCode()); + } + List dataSetList = sample.getDataSets().stream() + .map(DataSetWrapper::new) + .toList(); + dataSetAccumulator.addAll(dataSetList); + processedSampleCodes.add(sample.getCode()); + if (!dataSetList.isEmpty()) { + log.trace("added " + dataSetList.size() + " datasets for " + sample.getCode()); + } + sample.getChildren().forEach(it -> addAllDataSets(it, processedSampleCodes, dataSetAccumulator)); + + } + + +} diff --git a/src/main/java/life/qbic/qpostman/common/functions/SearchFiles.java b/src/main/java/life/qbic/qpostman/common/functions/SearchFiles.java new file mode 100644 index 0000000..e4a5d0f --- /dev/null +++ b/src/main/java/life/qbic/qpostman/common/functions/SearchFiles.java @@ -0,0 +1,120 @@ +package life.qbic.qpostman.common.functions; + +import static java.util.Objects.requireNonNull; + +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId; +import ch.ethz.sis.openbis.generic.dssapi.v3.IDataStoreServerApi; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.fetchoptions.DataSetFileFetchOptions; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.search.DataSetFileSearchCriteria; +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Stream; +import life.qbic.qpostman.common.structures.DataFile; +import life.qbic.qpostman.common.structures.DataSetWrapper; +import life.qbic.qpostman.openbis.OpenBisSessionProvider; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * Searches for data files based on a collection of DataSetWrapper objects. + * It utilizes a collection of IDataStoreServerApi instances to perform the search querying every datastore and aggregating the files. + */ +public class SearchFiles implements Function, Collection> { + + private static final Logger log = LogManager.getLogger(SearchFiles.class); + private final Collection dataStoreServerApis; + private final DataSetCounterUpdateListener dataSetCounterUpdateListener; + + public SearchFiles(Collection dataStoreServerApis, + DataSetCounterUpdateListener dataSetCounterUpdateListener) { + this.dataStoreServerApis = dataStoreServerApis; + this.dataSetCounterUpdateListener = dataSetCounterUpdateListener; + + } + + @Override + public Collection apply(Collection dataSetWrappers) { + return searchFiles(dataSetWrappers, dataSetCounterUpdateListener); + } + + @FunctionalInterface + public interface DataSetCounterUpdateListener { + + void updateCounter(int numberOfDatasets); + } + + + + public static class DataSetCounterProgressDisplay implements DataSetCounterUpdateListener { + + final int maxCount; + int currCount; + + public DataSetCounterProgressDisplay(int maxCount) { + this.maxCount = maxCount; + currCount = 0; + } + + @Override + public void updateCounter(int numberOfDatasets) { + currCount += numberOfDatasets; + System.out.printf("Indexing dataset %4s / %s\r", currCount, maxCount); + } + } + + + + private List searchFiles(Collection dataSets, + DataSetCounterUpdateListener updateListener) { + Stream dataSetFileQueries = dataSets.stream() + .map(DataSetWrapper::dataSetPermId) + .map(DataSetFileQuery::new); + Stream dataSetFiles = dataSetFileQueries + .peek(it -> updateListener.updateCounter(1)) + .flatMap(this::queryDataStoresForFiles); + Stream dataFiles = dataSetFiles + .map(dataSetFile -> { + DataSetPermId dataSetPermId = dataSetFile.getDataSetPermId(); + DataSetWrapper dataSet = dataSets.stream() + .filter(ds -> ds.dataSetPermId().equals(dataSetPermId)).findFirst() + .orElseThrow(); + return new DataFile(dataSetFile, dataSet); + }); + return dataFiles.toList(); + } + + private Stream queryDataStoresForFiles(DataSetFileQuery dataSetFileQuery) { + return dataStoreServerApis.stream() + .flatMap(dataStoreServerApi -> + { + List files = dataStoreServerApi.searchFiles(OpenBisSessionProvider.get().getToken(), + dataSetFileQuery.searchCriteria(), + dataSetFileQuery.fetchOptions()) + .getObjects(); + log.trace("Found " + files.size() + " files for " + + dataSetFileQuery.dataSetPermId.getPermId() + " on " + + dataStoreServerApi); + return files.stream(); + }) + .filter(file -> !file.isDirectory()); // filter out all folders but keeps the files + } + + private record DataSetFileQuery(DataSetPermId dataSetPermId) { + private DataSetFileQuery { + requireNonNull(dataSetPermId, "dataSetPermId must not be null"); + } + + public DataSetFileSearchCriteria searchCriteria() { + DataSetFileSearchCriteria criteria = new DataSetFileSearchCriteria(); + criteria.withDataSet().withPermId().thatEquals(dataSetPermId.getPermId()); + return criteria; + } + + public DataSetFileFetchOptions fetchOptions() { + return new DataSetFileFetchOptions(); + } + } + +} diff --git a/src/main/java/life/qbic/qpostman/common/functions/SortFiles.java b/src/main/java/life/qbic/qpostman/common/functions/SortFiles.java new file mode 100644 index 0000000..164a308 --- /dev/null +++ b/src/main/java/life/qbic/qpostman/common/functions/SortFiles.java @@ -0,0 +1,27 @@ +package life.qbic.qpostman.common.functions; + +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.function.Function; +import life.qbic.qpostman.common.structures.DataFile; + +public class SortFiles implements Function, List> { + + + private final Comparator comparator = Comparator + .comparing((DataFile dataFile) -> dataFile.dataSet().registrationTime()) + .reversed() + .thenComparing(DataFile::filePath, String::compareToIgnoreCase); + + @Override + public List apply(Collection dataFiles) { + return dataFiles.stream() + .sorted(comparator) + .toList(); + } + + public Comparator comparator() { + return comparator; + } +} diff --git a/src/main/java/life/qbic/qpostman/common/options/AuthenticationOptions.java b/src/main/java/life/qbic/qpostman/common/options/AuthenticationOptions.java new file mode 100644 index 0000000..e1bfe03 --- /dev/null +++ b/src/main/java/life/qbic/qpostman/common/options/AuthenticationOptions.java @@ -0,0 +1,72 @@ +package life.qbic.qpostman.common.options; + +import static java.util.Objects.nonNull; +import static picocli.CommandLine.ArgGroup; + +import java.util.StringJoiner; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import picocli.CommandLine.Option; + +public class AuthenticationOptions { + private static final Logger log = LogManager.getLogger(AuthenticationOptions.class); + + @Option( + names = {"-u", "--user"}, + required = true, + description = "openBIS user name") + public String user; + @ArgGroup(multiplicity = "1") // ensures the password is provided once with at least one of the possible options. + PasswordOptions passwordOptions; + + public char[] getPassword() { + return passwordOptions.getPassword(); + } + + /** + * official picocli documentation example + */ + static class PasswordOptions { + @Option(names = "--password:env", arity = "1", paramLabel = "", description = "provide the name of an environment variable to read in your password from") + protected String passwordEnvironmentVariable = ""; + + @Option(names = "--password:prop", arity = "1", paramLabel = "", description = "provide the name of a system property to read in your password from") + protected String passwordProperty = ""; + + @Option(names = "--password", arity = "0", description = "please provide your password", interactive = true) + protected char[] password = null; + + /** + * Gets the password. If no password is provided, the program exits. + * @return the password provided by the user. + */ + char[] getPassword() { + if (nonNull(password)) { + return password; + } + // System.getProperty(String key) does not work for empty or blank keys. + if (!passwordProperty.isBlank()) { + String systemProperty = System.getProperty(passwordProperty); + if (nonNull(systemProperty)) { + return systemProperty.toCharArray(); + } + } + String environmentVariable = System.getenv(passwordEnvironmentVariable); + if (nonNull(environmentVariable) && !environmentVariable.isBlank()) { + return environmentVariable.toCharArray(); + } + log.error("No password provided. Please provide your password."); + System.exit(2); + return null; // not reachable due to System.exit in previous line + } + + } + + @Override + public String toString() { + return new StringJoiner(", ", AuthenticationOptions.class.getSimpleName() + "[", "]") + .add("user='" + user + "'") + .toString(); + //ATTENTION: do not expose the password here! + } +} diff --git a/src/main/java/life/qbic/qpostman/common/options/FilterOptions.java b/src/main/java/life/qbic/qpostman/common/options/FilterOptions.java new file mode 100644 index 0000000..e406dc0 --- /dev/null +++ b/src/main/java/life/qbic/qpostman/common/options/FilterOptions.java @@ -0,0 +1,21 @@ +package life.qbic.qpostman.common.options; + +import java.util.ArrayList; +import java.util.List; +import java.util.StringJoiner; +import picocli.CommandLine; + +public class FilterOptions { + @CommandLine.Option(names = {"-s", "--suffix"}, + split = ",", + description= "only include files ending with one of these (case-insensitive) suffixes", + paramLabel = "") + public List suffixes = new ArrayList<>(0); + + @Override + public String toString() { + return new StringJoiner(", ", FilterOptions.class.getSimpleName() + "[", "]") + .add("suffixes=" + suffixes) + .toString(); + } +} diff --git a/src/main/java/life/qbic/qpostman/common/options/SampleIdentifierOptions.java b/src/main/java/life/qbic/qpostman/common/options/SampleIdentifierOptions.java new file mode 100644 index 0000000..d4e9a82 --- /dev/null +++ b/src/main/java/life/qbic/qpostman/common/options/SampleIdentifierOptions.java @@ -0,0 +1,96 @@ +package life.qbic.qpostman.common.options; + +import static picocli.CommandLine.ArgGroup; +import static picocli.CommandLine.Option; +import static picocli.CommandLine.Parameters; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.StringJoiner; +import java.util.stream.Collectors; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class SampleIdentifierOptions { + + private static final Logger log = LogManager.getLogger(SampleIdentifierOptions.class); + + @ArgGroup(multiplicity = "1") + public SampleInput sampleInput; + + public List getIds() { + return sampleInput.getIds(); + } + + @Override + public String toString() { + return new StringJoiner(", ", SampleIdentifierOptions.class.getSimpleName() + "[", "]") + .add("ids=" + getIds()) + .toString(); + } + + static class IdentityFileParser { + + public static List parseIdentityFile(Path path) { + File file = path.toFile(); + if (!file.exists()) { + log.error("File not found: " + file); + System.exit(2); + } + if (!file.canRead()) { + log.error("Not allowed to read file " + file); + System.exit(2); + } + try { + return Files.readAllLines(path).stream() + .filter(it -> !it.isBlank()) + .filter(it -> !it.startsWith("#")) + .map(String::strip) + .collect(Collectors.toList()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public static class SampleInput { + + @Option( + names = {"-f", "--file"}, + description = "a file with line-separated list of QBiC sample ids", + required = true) + public Path filePath; + + @Parameters(arity = "1..", paramLabel = "SAMPLE_IDENTIFIER", description = "one or more QBiC sample identifiers") + public List ids = new ArrayList<>(); + + public List getIds() { + if (ids.isEmpty()) { + List identities = IdentityFileParser.parseIdentityFile(filePath).stream() + .filter(it -> !it.isBlank()).toList(); + + if (identities.isEmpty()) { + log.error(filePath.toString() + " does not contain any identifiers."); + System.exit(2); + } + return identities; + } + return ids; + } + + + @Override + public String toString() { + return new StringJoiner(", ", SampleInput.class.getSimpleName() + "[", "]") + .add("filePath=" + filePath) + .add("ids=" + ids) + .toString(); + } + } + + +} diff --git a/src/main/java/life/qbic/qpostman/common/options/ServerOptions.java b/src/main/java/life/qbic/qpostman/common/options/ServerOptions.java new file mode 100644 index 0000000..f3ce7c5 --- /dev/null +++ b/src/main/java/life/qbic/qpostman/common/options/ServerOptions.java @@ -0,0 +1,47 @@ +package life.qbic.qpostman.common.options; + +import static picocli.CommandLine.Option; + +import java.util.ArrayList; +import java.util.List; +import java.util.StringJoiner; + +public class ServerOptions { + + @Option(names = {"--application-server"}, + paramLabel = "url", + description = "set the application server to find samples and datasets", + hidden = true) + public String as_url = "https://qbis.qbic.uni-tuebingen.de/openbis/openbis"; + + @Option(names = {"--datastore-server"}, + paramLabel = "url", + description = "add a data store server to find files on", + hidden = true) + public List dss_urls = new ArrayList<>(List.of( + "https://qbis.qbic.uni-tuebingen.de/datastore_server", + "https://qbis.qbic.uni-tuebingen.de/datastore_server2" + )); + + + @Option(names = {"--source-sample-type"}, + paramLabel = "Q_SAMPLE_TYPE", + description = "the sample type in openBis considered to be the source sample", + hidden = true) + public String sourceSampleType = "Q_TEST_SAMPLE"; + + @Option(names = {"--server-timeout"}, + paramLabel = "milliseconds", + description = "the server timeout in milliseconds", + hidden = true) + public long timeoutInMillis = 10_000; + + @Override + public String toString() { + + return new StringJoiner(", ", ServerOptions.class.getSimpleName() + "[", "]") + .add("as_url='" + as_url + "'") + .add("dss_urls=" + dss_urls) + .toString(); + } +} diff --git a/src/main/java/life/qbic/qpostman/common/structures/DataFile.java b/src/main/java/life/qbic/qpostman/common/structures/DataFile.java new file mode 100644 index 0000000..ae3fcca --- /dev/null +++ b/src/main/java/life/qbic/qpostman/common/structures/DataFile.java @@ -0,0 +1,43 @@ +package life.qbic.qpostman.common.structures; + +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.id.DataSetFilePermId; + +/** + * A file stored in our datastore. This class provides information about the sample the data is + * attached to as well as about the source of the attached data. + */ +public final class DataFile { + + private final DataSetFile file; + private final DataSetWrapper dataSet; + + public DataFile(DataSetFile dataSetFile, DataSetWrapper dataSet) { + this.file = dataSetFile; + this.dataSet = dataSet; + } + + public FileSize fileSize() { + return FileSize.of(file.getFileLength()); + } + + public String filePath() { + return file.getPath().replaceFirst("original/", ""); + } + + public DataSetFilePermId fileId() { + return file.getPermId(); + } + + public String fileName() { + return filePath().substring(filePath().lastIndexOf("/") + 1); + } + + public DataSetWrapper dataSet() { + return dataSet; + } + + public long crc32() { + return Integer.toUnsignedLong(file.getChecksumCRC32()); + } +} diff --git a/src/main/java/life/qbic/qpostman/common/structures/DataSetWrapper.java b/src/main/java/life/qbic/qpostman/common/structures/DataSetWrapper.java new file mode 100644 index 0000000..733ff62 --- /dev/null +++ b/src/main/java/life/qbic/qpostman/common/structures/DataSetWrapper.java @@ -0,0 +1,77 @@ +package life.qbic.qpostman.common.structures; + +import static java.util.Objects.nonNull; +import static java.util.Objects.requireNonNull; + +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; +import java.time.Instant; +import life.qbic.qpostman.common.functions.FindSourceSample; + +/** + * Wraps a DataSet as the openBis DTOs do not guarantee equals and hash code implementation. + */ +public final class DataSetWrapper { + + + private static FindSourceSample findSourceSample; + private final ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet dataSet; + private Sample sourceSample = null; + + public DataSetWrapper(ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet dataSet) { + requireNonNull(dataSet, "dataSet must not be null"); + this.dataSet = dataSet; + } + + public DataSetPermId dataSetPermId() { + return dataSet.getPermId(); + } + + public Instant registrationTime() { + return dataSet.getRegistrationDate().toInstant(); + } + + public String sampleCode() { + return dataSet.getSample().getCode(); + } + + /** + * DO NOT USE FOR EQUALS AND HASH CODE + * @return the sample where this dataset is attached + */ + public Sample sample() { + return dataSet.getSample(); + } + + public Sample sourceSample() { + if (nonNull(sourceSample)) { + return sourceSample; + } + this.sourceSample = findSourceSample.apply(sample()); + return sourceSample; + } + + public static void setFindSourceFunction(FindSourceSample findSourceSample) { + DataSetWrapper.findSourceSample = findSourceSample; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + DataSetWrapper that = (DataSetWrapper) o; + + return dataSetPermId().equals(that.dataSetPermId()) + && sampleCode().equals(that.sampleCode()) + && registrationTime().equals(that.registrationTime()); + } + + @Override + public int hashCode() { + int result = dataSetPermId().hashCode(); + result = 31 * result + sampleCode().hashCode(); + result = 31 * result + registrationTime().hashCode(); + return result; + } +} diff --git a/src/main/java/life/qbic/model/files/FileSize.java b/src/main/java/life/qbic/qpostman/common/structures/FileSize.java similarity index 58% rename from src/main/java/life/qbic/model/files/FileSize.java rename to src/main/java/life/qbic/qpostman/common/structures/FileSize.java index 1e9eedf..04182c1 100644 --- a/src/main/java/life/qbic/model/files/FileSize.java +++ b/src/main/java/life/qbic/qpostman/common/structures/FileSize.java @@ -1,4 +1,4 @@ -package life.qbic.model.files; +package life.qbic.qpostman.common.structures; /** * short description @@ -21,4 +21,12 @@ public static FileSize of(long bytes) { public long bytes() { return bytes; } + + public FileSize add(FileSize other) { + return add(this, other); + } + + public static FileSize add(FileSize size1, FileSize size2) { + return FileSize.of(size1.bytes() + size2.bytes()); + } } diff --git a/src/main/java/life/qbic/qpostman/download/DownloadCommand.java b/src/main/java/life/qbic/qpostman/download/DownloadCommand.java new file mode 100644 index 0000000..be1cb9e --- /dev/null +++ b/src/main/java/life/qbic/qpostman/download/DownloadCommand.java @@ -0,0 +1,138 @@ +package life.qbic.qpostman.download; + +import static picocli.CommandLine.Command; +import static picocli.CommandLine.Mixin; + +import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi; +import ch.ethz.sis.openbis.generic.dssapi.v3.IDataStoreServerApi; +import java.nio.file.Path; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import life.qbic.qpostman.common.AuthenticationException; +import life.qbic.qpostman.common.FileSizeFormatter; +import life.qbic.qpostman.common.functions.FileFilter; +import life.qbic.qpostman.common.functions.FindSourceSample; +import life.qbic.qpostman.common.functions.SearchDataSets; +import life.qbic.qpostman.common.functions.SearchFiles; +import life.qbic.qpostman.common.functions.SearchFiles.DataSetCounterProgressDisplay; +import life.qbic.qpostman.common.functions.SortFiles; +import life.qbic.qpostman.common.options.AuthenticationOptions; +import life.qbic.qpostman.common.options.FilterOptions; +import life.qbic.qpostman.common.options.SampleIdentifierOptions; +import life.qbic.qpostman.common.options.ServerOptions; +import life.qbic.qpostman.common.structures.DataFile; +import life.qbic.qpostman.common.structures.DataSetWrapper; +import life.qbic.qpostman.common.structures.FileSize; +import life.qbic.qpostman.download.WriteFileToDisk.DownloadReport; +import life.qbic.qpostman.openbis.ConnectionException; +import life.qbic.qpostman.openbis.OpenBisSessionProvider; +import life.qbic.qpostman.openbis.ServerFactory; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.remoting.RemoteAccessException; + +@Command(name = "download", + description = "Download data from QBiC.") +public class DownloadCommand implements Runnable { + private static final Logger log = LogManager.getLogger(DownloadCommand.class); + private static final String LOG_PATH = Optional.ofNullable(System.getProperty("log.path")) + .orElse("logs"); + @Mixin + AuthenticationOptions authenticationOptions; + @Mixin + SampleIdentifierOptions sampleIdentifierOptions; + @Mixin + FilterOptions filterOptions; + @Mixin + ServerOptions serverOptions; + @Mixin + DownloadOptions downloadOptions; + + @Override + public void run() { + try { + Functions functions = functions(); + + Collection dataSetFiles = functions.searchDataSets() + .andThen(it -> searchFiles(it).apply(it)) + .apply(sampleIdentifierOptions.getIds()); + + List sortedFiles = dataSetFiles.stream() + .filter(functions.fileFilter()) + .sorted(functions.sortFiles().comparator()) + .toList(); + + log.info( + "Downloading %s files (%s)".formatted(sortedFiles.size(), FileSizeFormatter.format( + FileSize.of(sortedFiles.stream().mapToLong(file -> file.fileSize().bytes() + ).sum()), 6))); + List downloadReports = sortedFiles.stream() + .map(functions.writeFileToDisk()) + .peek(downloadReport -> { + if (downloadReport.isSuccess()) { + log.info("Download successful for " + downloadReport.outputPath()); + } else { + log.warn("Failed to download " + downloadReport.outputPath()); + } + }) + .toList(); + List successfulDownloads = downloadReports.stream() + .filter(DownloadReport::isSuccess).toList(); + List failedDownloads = downloadReports.stream() + .filter(DownloadReport::isFailure).toList(); + log.info("Successfully downloaded " + successfulDownloads.size() +" files."); + if (!failedDownloads.isEmpty()) { + log.warn("Failed to download %s / %s files.".formatted(failedDownloads.size(), downloadReports.size())); + } + + + } catch (RemoteAccessException remoteAccessException) { + log.error( + "Failed to connect to OpenBis: " + remoteAccessException.getCause().getMessage()); + log.debug(remoteAccessException.getMessage(), remoteAccessException); + System.exit(1); + } catch (AuthenticationException authenticationException) { + log.error( + "Could not authenticate user %s. Please make sure to provide the correct password.".formatted( + authenticationException.username())); + log.debug(authenticationException.getMessage(), authenticationException); + System.exit(1); + } catch (ConnectionException e) { + log.error("Could not connect to QBiC's data source. Have you requested access to the " + + "server? If not please write to support@qbic.zendesk.com"); + log.debug(e.getMessage(), e); + System.exit(1); + } catch (RuntimeException e) { + log.error("Something went wrong. For more detailed output see " + Path.of(LOG_PATH, "postman.log").toAbsolutePath()); + log.debug(e.getMessage(), e); + } + } + + private SearchFiles searchFiles(Collection it) { + return new SearchFiles(dataStoreServerApis(), new DataSetCounterProgressDisplay(it.size())); + } + + private Collection dataStoreServerApis() { + return ServerFactory.dataStoreServers(serverOptions.dss_urls, + serverOptions.timeoutInMillis); + } + + private Functions functions() { + IApplicationServerApi applicationServerApi = ServerFactory.applicationServer(serverOptions.as_url, serverOptions.timeoutInMillis); + OpenBisSessionProvider.init(applicationServerApi, authenticationOptions.user, new String(authenticationOptions.getPassword())); + SearchDataSets searchDataSets = new SearchDataSets(applicationServerApi); + FileFilter myAwesomeFileFilter = FileFilter.create().withSuffixes(filterOptions.suffixes); + WriteFileToDisk writeFileToDisk = new WriteFileToDisk(dataStoreServerApis().toArray(IDataStoreServerApi[]::new)[0], + downloadOptions.bufferSize, Path.of(downloadOptions.outputPath), downloadOptions.successiveDownloadAttempts); + FindSourceSample findSourceSample = new FindSourceSample(serverOptions.sourceSampleType); + SortFiles sortFiles = new SortFiles(); + DataSetWrapper.setFindSourceFunction(findSourceSample); + + return new Functions(searchDataSets, writeFileToDisk, sortFiles, myAwesomeFileFilter); + } + + private record Functions(SearchDataSets searchDataSets, WriteFileToDisk writeFileToDisk, SortFiles sortFiles, FileFilter fileFilter) { + + } +} diff --git a/src/main/java/life/qbic/qpostman/download/DownloadOptions.java b/src/main/java/life/qbic/qpostman/download/DownloadOptions.java new file mode 100644 index 0000000..675bdfa --- /dev/null +++ b/src/main/java/life/qbic/qpostman/download/DownloadOptions.java @@ -0,0 +1,36 @@ +package life.qbic.qpostman.download; + +import static picocli.CommandLine.Help; +import static picocli.CommandLine.Option; + +import java.util.Optional; +import java.util.StringJoiner; + +public class DownloadOptions { + @Option(names = {"--buffer-size"}, + defaultValue = "1024", + showDefaultValue = Help.Visibility.ALWAYS, + description = + "dataset download performance can be improved by increasing this value with a multiple of 1024 (default)." + + " Only change this if you know what you are doing.", + hidden = true) + public int bufferSize; + + @Option(names = {"-o", "--output-dir"}, + description = "specify where to write the downloaded data") + public String outputPath = Optional.ofNullable(System.getenv("user.dir")).orElse("."); + + @Option(names = "--download-attempts", + defaultValue = "3", + description = "how often to attempt file download in case the download failed", + hidden = true) + public int successiveDownloadAttempts; + + @Override + public String toString() { + return new StringJoiner(", ", DownloadOptions.class.getSimpleName() + "[", "]") + .add("bufferSize=" + bufferSize) + .add("outputPath='" + outputPath + "'") + .toString(); + } +} diff --git a/src/main/java/life/qbic/qpostman/download/DownloadProgressListener.java b/src/main/java/life/qbic/qpostman/download/DownloadProgressListener.java new file mode 100644 index 0000000..2c6e9af --- /dev/null +++ b/src/main/java/life/qbic/qpostman/download/DownloadProgressListener.java @@ -0,0 +1,26 @@ +package life.qbic.qpostman.download; + +import life.qbic.qpostman.common.ProgressBar; + +/** + * The DownloadProgressListener class provides methods to track the progress of a file download. + * It uses a ProgressBar object to display the download progress. + */ +public class DownloadProgressListener implements WriteProgressListener { + + private final ProgressBar progressBar; + + public DownloadProgressListener(String fileName, long totalFileSize) { + progressBar = new ProgressBar(fileName, totalFileSize); + } + + @Override + public void update(long bytesWritten) { + progressBar.updateProgress(bytesWritten); + } + + @Override + public void finish() { + progressBar.remove(); + } +} diff --git a/src/main/java/life/qbic/qpostman/download/WriteFileToDisk.java b/src/main/java/life/qbic/qpostman/download/WriteFileToDisk.java new file mode 100644 index 0000000..0e8deda --- /dev/null +++ b/src/main/java/life/qbic/qpostman/download/WriteFileToDisk.java @@ -0,0 +1,157 @@ +package life.qbic.qpostman.download; + +import ch.ethz.sis.openbis.generic.dssapi.v3.IDataStoreServerApi; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownloadOptions; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownloadReader; +import java.io.BufferedWriter; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Collections; +import java.util.function.Function; +import life.qbic.qpostman.common.structures.DataFile; +import life.qbic.qpostman.download.WriteFileToDisk.DownloadReport; +import life.qbic.qpostman.openbis.OpenBisSessionProvider; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * A function writing a DataFile to disk and returning the write report. + */ +public class WriteFileToDisk implements Function { + + private static final String LOG_PATH = System.getProperty("log.path", "logs"); + + private final IDataStoreServerApi dataStoreServerApi; + private final int bufferSize; + private final Path outputDirectory; + private final int downloadAttempts; + + private static final Logger log = LogManager.getLogger(WriteFileToDisk.class); + public WriteFileToDisk(IDataStoreServerApi dataStoreServerApi, int bufferSize, Path outputDirectory, + int downloadAttempts) { + this.dataStoreServerApi = dataStoreServerApi; + this.bufferSize = bufferSize; + this.outputDirectory = outputDirectory; + this.downloadAttempts = downloadAttempts; + } + + private static Path toOutputPath(DataFile dataFile, Path outputDirectory) { + String originalFilePath = dataFile.filePath(); + Path outFilePath = outputDirectory.resolve(originalFilePath); + return outFilePath; + } + + private AutoClosableDataSetFileDownloadReader toReader(DataFile dataFile) { + return new AutoClosableDataSetFileDownloadReader( + dataStoreServerApi.downloadFiles(OpenBisSessionProvider.get().getToken(), + Collections.singletonList(dataFile.fileId()), + new DataSetFileDownloadOptions())); + } + + private InputStream toInputStream(DataSetFileDownloadReader reader) { + return reader.read().getInputStream(); + } + + /** + * Download the specific data file. + * + * @param dataFile the data file to be applied + * @return the download report + */ + @Override + public DownloadReport apply(DataFile dataFile) { + int bufferSize = + (dataFile.fileSize().bytes() < this.bufferSize) ? (int) dataFile.fileSize().bytes() + : this.bufferSize; + Path outputPath = toOutputPath(dataFile, outputDirectory); + if (WriteUtils.doesExistWithCrc32(outputPath, dataFile.crc32(), bufferSize)) { + log.info("File " + outputPath + " exists on your machine."); + return new DownloadReport(dataFile.crc32(), dataFile.crc32(), outputPath.toAbsolutePath()); + } + DownloadReport downloadReport = null; + for (int attempt = 1; attempt <= downloadAttempts; attempt++) { + downloadReport = writeToDisk(dataFile, new DownloadProgressListener(dataFile.fileName(), dataFile.fileSize().bytes())); + + if (downloadReport.isSuccess()) { + return downloadReport; + } else { + log.warn("Download attempt %s / %s failed for %s".formatted(attempt, downloadAttempts, downloadReport.outputPath())); + log.trace(downloadReport); + } + } + try { + Path file = Path.of(LOG_PATH, "checksum-mismatch.log"); + assert downloadReport != null : "download report is null"; + Files.writeString(file, downloadReport + "\n", StandardOpenOption.CREATE, + StandardOpenOption.APPEND); + } catch (IOException e) { + throw new RuntimeException(e); + } + return downloadReport; + } + + public record DownloadReport(long expectedCrc32, long actualCrc32, Path outputPath) { + public boolean isSuccess() { + return expectedCrc32 == actualCrc32; + } + public boolean isFailure() { + return !isSuccess(); + } + + @Override + public String toString() { + return "%s\t%s\t%s".formatted(Long.toHexString(expectedCrc32()), + Long.toHexString(actualCrc32()), + outputPath()); + } + } + + private DownloadReport writeToDisk(DataFile dataFile, WriteProgressListener progressListener) { + Path outFile = toOutputPath(dataFile, outputDirectory); + + Path crc32File = Path.of(outFile.toAbsolutePath() + ".crc32"); + try { + Files.createDirectories(outFile.getParent()); + } catch (IOException e) { + throw new RuntimeException(e); + } + try (AutoClosableDataSetFileDownloadReader reader = toReader(dataFile); //we need to reader here, so it is closed correctly + InputStream inputStream = toInputStream(reader); + FileOutputStream outputStream = new FileOutputStream(outFile.toFile()); + BufferedWriter crc32FileWriter = new BufferedWriter( + new FileWriter(crc32File.toFile()))) { + int bufferSize = + (dataFile.fileSize().bytes() < this.bufferSize) ? (int) dataFile.fileSize().bytes() + : this.bufferSize; + long writtenCrc32 = WriteUtils.write(bufferSize, inputStream, outputStream, + progressListener); + if (writtenCrc32 != dataFile.crc32()) { + return new DownloadReport(dataFile.crc32(), writtenCrc32, outFile.toAbsolutePath()); + } + try { + Files.createDirectories(crc32File.getParent()); + } catch (IOException e) { + throw new RuntimeException(e); + } + crc32FileWriter.write(Long.toHexString(writtenCrc32) + "\t" + dataFile.fileName()); + crc32FileWriter.flush(); + return new DownloadReport(dataFile.crc32(), writtenCrc32, outFile.toAbsolutePath()); + } catch (IOException e) { + log.error(e.getMessage(), e); + return new DownloadReport(dataFile.crc32(), 0, outFile.toAbsolutePath()); + } + } + // note: DataSetFileDownloadReader closes the input stream after it finished reading it. + private static class AutoClosableDataSetFileDownloadReader extends DataSetFileDownloadReader implements AutoCloseable { + public AutoClosableDataSetFileDownloadReader(InputStream in) { + super(in); + } + + + } +} diff --git a/src/main/java/life/qbic/qpostman/download/WriteProgressListener.java b/src/main/java/life/qbic/qpostman/download/WriteProgressListener.java new file mode 100644 index 0000000..313b28c --- /dev/null +++ b/src/main/java/life/qbic/qpostman/download/WriteProgressListener.java @@ -0,0 +1,23 @@ +package life.qbic.qpostman.download; + +/** + * The WriteProgressListener interface provides methods for receiving updates on the progress + * of a write operation. + */ +public interface WriteProgressListener { + + /** + * Updates the number of bytes written. + * + * @param bytesWritten the number of bytes written to be updated + */ + void update(long bytesWritten); + + /** + * Finishes the operation. + *

+ * This method restores the original state of the writer. + *

+ */ + void finish(); +} diff --git a/src/main/java/life/qbic/qpostman/download/WriteUtils.java b/src/main/java/life/qbic/qpostman/download/WriteUtils.java new file mode 100644 index 0000000..3170938 --- /dev/null +++ b/src/main/java/life/qbic/qpostman/download/WriteUtils.java @@ -0,0 +1,79 @@ +package life.qbic.qpostman.download; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; +import java.util.zip.CRC32; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * Utility class for reading and writing data files. + */ +public class WriteUtils { + + private static final Logger log = LogManager.getLogger(WriteUtils.class); + + public static long write(int bufferSize, InputStream inputStream, OutputStream outputStream, + WriteProgressListener progressListener) + throws IOException { + CRC32 crc32 = new CRC32(); + byte[] buffer = new byte[bufferSize]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) > 0) { + crc32.update(buffer, 0, bytesRead); + outputStream.write(buffer, 0, bytesRead); + progressListener.update(bytesRead); + outputStream.flush(); + } + progressListener.finish(); + return crc32.getValue(); + } + + public static long readCrc32(Path file, int bufferSize) { + if (!file.toFile().exists()) { + throw new IllegalArgumentException("File " + file.toAbsolutePath() + " was expected but not found."); + } + return readCrc32FromFile(file).orElseGet(() -> calculateCrc32(file, bufferSize)); + } + + public static boolean doesExistWithCrc32(Path file, long expectedCrc32, int bufferSize) { + return file.toFile().exists() && expectedCrc32 == readCrc32(file, bufferSize); + } + + private static long calculateCrc32(Path file, int bufferSize) { + byte[] buffer = new byte[bufferSize]; + try (InputStream inputStream = new FileInputStream(file.toFile())) { + int bytesRead; + CRC32 crc32 = new CRC32(); + while ((bytesRead = inputStream.read(buffer)) > 0) { + crc32.update(buffer, 0, bytesRead); + } + return crc32.getValue(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static Optional readCrc32FromFile(Path file) { + Path crc32FileName = Path.of(file.toAbsolutePath() + ".crc32"); + + if (!crc32FileName.toFile().exists()) { + return Optional.empty(); + } + + try (BufferedReader reader = Files.newBufferedReader(crc32FileName)) { + String firstLine = reader.readLine(); + return Optional.ofNullable(firstLine) + .map(line -> Long.parseLong(line.split("\\s+")[0], 16)); + } catch (IOException e) { + log.warn("Could not open " + crc32FileName.getFileName()); + return Optional.empty(); + } + } +} diff --git a/src/main/java/life/qbic/qpostman/list/DataFileTableFormatter.java b/src/main/java/life/qbic/qpostman/list/DataFileTableFormatter.java new file mode 100644 index 0000000..2c3cbcc --- /dev/null +++ b/src/main/java/life/qbic/qpostman/list/DataFileTableFormatter.java @@ -0,0 +1,74 @@ +package life.qbic.qpostman.list; + +import static java.util.Objects.isNull; +import static java.util.Objects.requireNonNull; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import life.qbic.qpostman.common.FileSizeFormatter; +import life.qbic.qpostman.common.structures.DataFile; + +/** + * The DataFileTableFormatter class is responsible for formatting a list of DataFiles + * into a table format using specified delimiter and column settings. + */ +public class DataFileTableFormatter { + + private final List> columns; + + public DataFileTableFormatter(boolean exactFileSize, boolean withChecksum) { + columns = new ArrayList<>(); + + columns.add(Column.create("Dataset", + file -> file.dataSet().sampleCode() + " (" + file.dataSet().dataSetPermId().getPermId() + + ")")); + columns.add(Column.create("Source", file -> file.dataSet().sourceSample().getCode())); + columns.add(Column.create("Registration", file -> file.dataSet().registrationTime().toString())); + columns.add(Column.create("Size", file -> exactFileSize + ? String.valueOf(file.fileSize().bytes()) + : FileSizeFormatter.format(file.fileSize(), 6))); + if (withChecksum) { + columns.add(Column.create("CRC32", file -> Long.toHexString(file.crc32()))); + } + columns.add(Column.create("File", DataFile::filePath)); + } + + public String formatAsTable(List files, String delimiter, boolean withHeader) { + StringBuilder result = new StringBuilder(); + if (withHeader) { + List columnNames = columns.stream() + .map(Column::name) + .toList(); + String headerRow = String.join(delimiter, columnNames) + "\n"; + result.append(headerRow); + } + files.stream() + .map(file -> toRow(file, delimiter)) + .forEach(result::append); + return result.toString(); + } + + private String toRow(DataFile file, String delimiter) { + List values = columns.stream().map(c -> c.toValue(file)).toList(); + return String.join(delimiter, values) + "\n"; + } + + private record Column(String name, Function valueProvider) { + + Column { + requireNonNull(valueProvider, "valueProvider must not be null"); + if (isNull(name)) { + name = ""; + } + } + + public static Column create(String name, Function valueProvider) { + return new Column<>(name, valueProvider); + } + + public T toValue(DataFile dataFile) { + return valueProvider.apply(dataFile); + } + } +} diff --git a/src/main/java/life/qbic/qpostman/list/LegacyOutputFormatter.java b/src/main/java/life/qbic/qpostman/list/LegacyOutputFormatter.java new file mode 100644 index 0000000..24b406d --- /dev/null +++ b/src/main/java/life/qbic/qpostman/list/LegacyOutputFormatter.java @@ -0,0 +1,82 @@ +package life.qbic.qpostman.list; + +import static java.util.Objects.requireNonNull; + +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId; +import java.util.List; +import life.qbic.qpostman.common.FileSizeFormatter; +import life.qbic.qpostman.common.structures.DataFile; +import life.qbic.qpostman.common.structures.DataSetWrapper; +import life.qbic.qpostman.common.structures.FileSize; + +/** + * Formats a list of data files in legacy format to support backwards compatibility. + */ +public class LegacyOutputFormatter { + + public String format(DataSetSummary dataSetSummary, boolean exactFileSize, boolean withChecksum) { + String summaryOutput = """ + # Dataset %s + # Source %s + # Registration %s + # Size %s + """.formatted( + dataSetSummary.datasetName(), + dataSetSummary.sourceSampleCode(), + dataSetSummary.dataSet().registrationTime(), + exactFileSize ? dataSetSummary.totalSize().bytes() : FileSizeFormatter.format(dataSetSummary.totalSize() + )); + StringBuilder result = new StringBuilder(summaryOutput); + for (DataFile datafile : dataSetSummary.datafiles()) { + String fileOutput = withChecksum + ? "%s\t%s\t%s".formatted(exactFileSize ? datafile.fileSize().bytes() + : FileSizeFormatter.format(datafile.fileSize(), 6), + Long.toHexString(datafile.crc32()), + datafile.fileName()) + : "%s\t%s".formatted( + exactFileSize ? datafile.fileSize().bytes() : FileSizeFormatter.format(datafile.fileSize(), 6), + datafile.fileName()) ; + result.append(fileOutput); + result.append("\n"); + } + return result.toString(); + } + + public record DataSetSummary(List datafiles) { + + public DataSetSummary { + requireNonNull(datafiles, "dataSetFiles must not be null"); + if (datafiles.isEmpty()) { + throw new IllegalArgumentException("No data files provided."); + } + DataSetPermId dataSetPermId = datafiles.get(0).dataSet().dataSetPermId(); + if (datafiles.stream().anyMatch(file -> !file.dataSet().dataSetPermId().equals(dataSetPermId))) { + throw new IllegalArgumentException("Not all files are of the same dataset."); + } + } + + private DataSetWrapper dataSet() { + return datafiles.get(0).dataSet(); + } + + public String sourceSampleCode() { + return dataSet().sourceSample().getCode(); + } + + public String datasetName() { + DataSetWrapper dataSet = dataSet(); + return "%s (%s)".formatted(dataSet.sampleCode(), dataSet.dataSetPermId().getPermId()); + } + + public FileSize totalSize() { + return datafiles.stream() + .map(DataFile::fileSize) + .reduce((size, size2) -> FileSize.add(size, size2)) + .orElse(FileSize.of(0)); + } + + + + } + +} diff --git a/src/main/java/life/qbic/qpostman/list/ListCommand.java b/src/main/java/life/qbic/qpostman/list/ListCommand.java new file mode 100644 index 0000000..2cbe352 --- /dev/null +++ b/src/main/java/life/qbic/qpostman/list/ListCommand.java @@ -0,0 +1,138 @@ +package life.qbic.qpostman.list; + +import static picocli.CommandLine.Command; +import static picocli.CommandLine.Mixin; + +import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi; +import ch.ethz.sis.openbis.generic.dssapi.v3.IDataStoreServerApi; +import java.nio.file.Path; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import life.qbic.qpostman.common.AuthenticationException; +import life.qbic.qpostman.common.functions.FileFilter; +import life.qbic.qpostman.common.functions.FindSourceSample; +import life.qbic.qpostman.common.functions.SearchDataSets; +import life.qbic.qpostman.common.functions.SearchFiles; +import life.qbic.qpostman.common.functions.SortFiles; +import life.qbic.qpostman.common.options.AuthenticationOptions; +import life.qbic.qpostman.common.options.FilterOptions; +import life.qbic.qpostman.common.options.SampleIdentifierOptions; +import life.qbic.qpostman.common.options.ServerOptions; +import life.qbic.qpostman.common.structures.DataFile; +import life.qbic.qpostman.common.structures.DataSetWrapper; +import life.qbic.qpostman.list.LegacyOutputFormatter.DataSetSummary; +import life.qbic.qpostman.openbis.ConnectionException; +import life.qbic.qpostman.openbis.OpenBisSessionProvider; +import life.qbic.qpostman.openbis.ServerFactory; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.remoting.RemoteAccessException; + +@Command(name = "list", + description = "lists all the datasets found for the given identifiers") +public class ListCommand implements Runnable { + private static final Logger log = LogManager.getLogger(ListCommand.class); + private static final String LOG_PATH = Optional.ofNullable(System.getProperty("log.path")) + .orElse("logs"); + @Mixin + AuthenticationOptions authenticationOptions; + @Mixin + SampleIdentifierOptions sampleIdentifierOptions; + @Mixin + FilterOptions filterOptions; + @Mixin + ServerOptions serverOptions; + @Mixin + ListOptions listOptions; + + @Override + public void run() { + try { + Functions functions = setupFunctions(); + + Collection dataSetFiles = functions.searchDataSets() + .andThen(functions.searchFiles()) + .apply(sampleIdentifierOptions.getIds()); + + List processedFiles = dataSetFiles.stream() + .filter(functions.fileFilter()) + .sorted(functions.sortFiles().comparator()) + .toList(); + + Consumer> output = switch (listOptions.outputFormat) { + case LEGACY -> this::listAsLegacy; + case TSV -> this::listAsTsv; + }; + output.accept(processedFiles); + + } catch (RemoteAccessException remoteAccessException) { + log.error( + "Failed to connect to OpenBis: " + remoteAccessException.getCause().getMessage()); + log.debug(remoteAccessException.getMessage(), remoteAccessException); + System.exit(1); + } catch (AuthenticationException authenticationException) { + log.error( + "Could not authenticate user %s. Please make sure to provide the correct password.".formatted( + authenticationException.username())); + log.debug(authenticationException.getMessage(), authenticationException); + System.exit(1); + } catch (ConnectionException e) { + log.error("Could not connect to QBiC's data source. Have you requested access to the " + + "server? If not please write to support@qbic.zendesk.com"); + log.debug(e.getMessage(), e); + System.exit(1); + } catch (RuntimeException e) { + log.error("Something went wrong. For more detailed output see " + Path.of(LOG_PATH, + "postman.log").toAbsolutePath()); + log.debug(e.getMessage(), e); + } + } + + private void listAsTsv(List processedFiles) { + boolean withHeader = !listOptions.withoutHeader; + DataFileTableFormatter dataFileTableFormatter = new DataFileTableFormatter(listOptions.exactFilesize, listOptions.withChecksum); + String tsvContent = dataFileTableFormatter + .formatAsTable(processedFiles, "\t", withHeader); + System.out.println(tsvContent); + } + + private void listAsLegacy(List processedFiles) { + LegacyOutputFormatter legacyOutputFormatter = new LegacyOutputFormatter(); + Map> groupedFiles = processedFiles.stream() + .collect(Collectors.groupingBy(DataFile::dataSet)); + for (Entry> dataSetWrapperListEntry : groupedFiles.entrySet()) { + String output = legacyOutputFormatter.format( + new DataSetSummary(dataSetWrapperListEntry.getValue()), + listOptions.exactFilesize, listOptions.withChecksum); + System.out.println(output); + } + } + + private Functions setupFunctions() { + IApplicationServerApi applicationServerApi = ServerFactory.applicationServer( + serverOptions.as_url, serverOptions.timeoutInMillis); + OpenBisSessionProvider.init(applicationServerApi, authenticationOptions.user, + new String(authenticationOptions.getPassword())); + Collection dataStoreServerApis = ServerFactory.dataStoreServers( + serverOptions.dss_urls, serverOptions.timeoutInMillis); + SearchDataSets searchDataSets = new SearchDataSets(applicationServerApi); + FileFilter myAwesomeFileFilter = FileFilter.create() + .withSuffixes(filterOptions.suffixes); + SearchFiles searchFiles = new SearchFiles(dataStoreServerApis, number -> {}); + FindSourceSample findSourceSample = new FindSourceSample(serverOptions.sourceSampleType); + + SortFiles sortFiles = new SortFiles(); + + DataSetWrapper.setFindSourceFunction(findSourceSample); + return new Functions(searchDataSets, myAwesomeFileFilter, searchFiles, sortFiles); + } + + private record Functions(SearchDataSets searchDataSets, FileFilter fileFilter, SearchFiles searchFiles, SortFiles sortFiles) { + + } +} diff --git a/src/main/java/life/qbic/qpostman/list/ListOptions.java b/src/main/java/life/qbic/qpostman/list/ListOptions.java new file mode 100644 index 0000000..b92381d --- /dev/null +++ b/src/main/java/life/qbic/qpostman/list/ListOptions.java @@ -0,0 +1,38 @@ +package life.qbic.qpostman.list; + +import java.util.StringJoiner; +import picocli.CommandLine.Help.Visibility; +import picocli.CommandLine.Option; + +public class ListOptions { + @Option(names = "--with-checksum", defaultValue = "false", + description = "list the crc32 checksum for each file") + public boolean withChecksum; + + @Option(names = "--exact-filesize", defaultValue = "false", + description = "use exact byte count instead of unit suffixes: Byte, Kilobyte, Megabyte, Gigabyte, Terabyte and Petabyte using base 2 for sizes.", + showDefaultValue = Visibility.ON_DEMAND) + public boolean exactFilesize; + + @Option(names = "--format", defaultValue = "LEGACY", + converter = OutputFormat.OutputFormatConverter.class, + completionCandidates = OutputFormat.CompletionCandidates.class, + paramLabel = "", + description = "The format to list files in. Case-insensitive. Possible values: ${COMPLETION-CANDIDATES}", + showDefaultValue = Visibility.ALWAYS, + required = true) + public OutputFormat outputFormat; + + @Option(names = "--without-header", defaultValue = "false", + description = "remove the header line from the output. Only takes effect for tabular output formats.", + showDefaultValue = Visibility.ON_DEMAND) + public boolean withoutHeader; + + @Override + public String toString(){ + return new StringJoiner(", ", ListOptions.class.getSimpleName() + "[", "]") + .add("withChecksum=" + withChecksum) + .add("exactFilesize=" + exactFilesize) + .toString(); + } +} diff --git a/src/main/java/life/qbic/qpostman/list/OutputFormat.java b/src/main/java/life/qbic/qpostman/list/OutputFormat.java new file mode 100644 index 0000000..1a42c7b --- /dev/null +++ b/src/main/java/life/qbic/qpostman/list/OutputFormat.java @@ -0,0 +1,32 @@ +package life.qbic.qpostman.list; + +import java.util.Arrays; +import java.util.Iterator; +import picocli.CommandLine.ITypeConverter; + +public enum OutputFormat { + LEGACY, + TSV; + + public static class OutputFormatConverter implements ITypeConverter { + + @Override + public OutputFormat convert(String input) { + return Arrays.stream(OutputFormat.values()) + .map(Enum::name) + .filter(name -> name.equalsIgnoreCase(input)) + .map(OutputFormat::valueOf) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("Unknown format " + input)); + } + } + + public static class CompletionCandidates implements Iterable { + + @Override + public Iterator iterator() { + return Arrays.stream(values()).map(Enum::name).iterator(); + } + } + +} diff --git a/src/main/java/life/qbic/model/download/ConnectionException.java b/src/main/java/life/qbic/qpostman/openbis/ConnectionException.java similarity index 79% rename from src/main/java/life/qbic/model/download/ConnectionException.java rename to src/main/java/life/qbic/qpostman/openbis/ConnectionException.java index 73bf9af..32d5c83 100644 --- a/src/main/java/life/qbic/model/download/ConnectionException.java +++ b/src/main/java/life/qbic/qpostman/openbis/ConnectionException.java @@ -1,14 +1,10 @@ -package life.qbic.model.download; +package life.qbic.qpostman.openbis; /** * ConnectionException indicates issues when a client wants to connect with the openBIS API. */ public class ConnectionException extends RuntimeException { - ConnectionException() { - super(); - } - ConnectionException(String msg) { super(msg); } diff --git a/src/main/java/life/qbic/qpostman/openbis/OpenBisSession.java b/src/main/java/life/qbic/qpostman/openbis/OpenBisSession.java new file mode 100644 index 0000000..82d38dd --- /dev/null +++ b/src/main/java/life/qbic/qpostman/openbis/OpenBisSession.java @@ -0,0 +1,48 @@ +package life.qbic.qpostman.openbis; + +import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi; +import java.util.Objects; +import life.qbic.qpostman.common.AuthenticationException; + +public class OpenBisSession { + private final String username; + private final String password; + private final IApplicationServerApi applicationServerApi; + private String token; + + public OpenBisSession(IApplicationServerApi applicationServerApi, String username, String password) { + this.applicationServerApi = applicationServerApi; + this.username = username; + this.password = password; + token = login(); + } + + public void logout() { + applicationServerApi.logout(token); + token = null; + } + + public boolean isLoggedIn() { + return Objects.nonNull(token) && !token.isBlank() && applicationServerApi.isSessionActive(token); + } + + public String getToken() { + return isLoggedIn() ? token : login(); + } + + private String login() throws AuthenticationException { + if (isLoggedIn()) { + logout(); + } + try { + token = applicationServerApi.login(username, password); + } catch (Exception e) { + throw new ConnectionException("Connection to openBIS server failed.", e); + } + if (Objects.isNull(token)) { + throw new AuthenticationException("openbis application server did not produce a session token for " + username, username); + } + return token; + } + +} diff --git a/src/main/java/life/qbic/qpostman/openbis/OpenBisSessionProvider.java b/src/main/java/life/qbic/qpostman/openbis/OpenBisSessionProvider.java new file mode 100644 index 0000000..4bc72d2 --- /dev/null +++ b/src/main/java/life/qbic/qpostman/openbis/OpenBisSessionProvider.java @@ -0,0 +1,21 @@ +package life.qbic.qpostman.openbis; + +import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi; +import java.util.Objects; + +public class OpenBisSessionProvider { + + private static OpenBisSession openBisSession; + + public static OpenBisSession init(IApplicationServerApi applicationServerApi, String username, String password) { + openBisSession = new OpenBisSession(applicationServerApi, username, password); + return openBisSession; + } + + public static OpenBisSession get() { + if (Objects.isNull(openBisSession)) { + throw new RuntimeException("Session not initialized"); + } + return openBisSession; + } +} diff --git a/src/main/java/life/qbic/qpostman/openbis/ServerFactory.java b/src/main/java/life/qbic/qpostman/openbis/ServerFactory.java new file mode 100644 index 0000000..8135913 --- /dev/null +++ b/src/main/java/life/qbic/qpostman/openbis/ServerFactory.java @@ -0,0 +1,25 @@ +package life.qbic.qpostman.openbis; + +import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi; +import ch.ethz.sis.openbis.generic.dssapi.v3.IDataStoreServerApi; +import ch.systemsx.cisd.common.spring.HttpInvokerUtils; +import java.util.Collection; +import java.util.List; + +/** + * Creates server instances given urls + */ +public class ServerFactory { + public static Collection dataStoreServers(List dataStoreServerUrls, long serverTimeoutInMillis) { + return dataStoreServerUrls.stream() + .filter(dataStoreServerUrl -> !dataStoreServerUrl.isEmpty()) + .map(dataStoreServerUrl -> HttpInvokerUtils.createStreamSupportingServiceStub(IDataStoreServerApi.class, + dataStoreServerUrl + IDataStoreServerApi.SERVICE_URL, serverTimeoutInMillis)) + .toList(); + } + + public static IApplicationServerApi applicationServer(String applicationServerUrl, long serverTimeoutInMillis) { + return HttpInvokerUtils.createServiceStub(IApplicationServerApi.class, + applicationServerUrl + IApplicationServerApi.SERVICE_URL, serverTimeoutInMillis); + } +} diff --git a/src/main/java/life/qbic/util/StringUtil.java b/src/main/java/life/qbic/util/StringUtil.java deleted file mode 100644 index a2c8fa4..0000000 --- a/src/main/java/life/qbic/util/StringUtil.java +++ /dev/null @@ -1,9 +0,0 @@ -package life.qbic.util; - -public class StringUtil { - - public static boolean endsWithIgnoreCase(String input, String suffix) { - int suffixLength = suffix.length(); - return input.regionMatches(true, input.length() - suffixLength, suffix, 0, suffixLength); - } -} diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml index 282f112..3ab8674 100644 --- a/src/main/resources/log4j2.xml +++ b/src/main/resources/log4j2.xml @@ -1,17 +1,25 @@ + + $${sys:log.level:-$${env:LOG_LEVEL:-info}} + $${sys:log.path:-logs} + - - + + + + + - - - + + + + diff --git a/src/test/groovy/FailingSpockTest.groovy b/src/test/groovy/FailingSpockTest.groovy new file mode 100644 index 0000000..e133a0e --- /dev/null +++ b/src/test/groovy/FailingSpockTest.groovy @@ -0,0 +1,13 @@ +import spock.lang.Ignore +import spock.lang.Specification + +/** + * Use this test to see whether maven executes Spock tests or not. + */ +@Ignore +class FailingSpockTest extends Specification { + def "this test always fails"() { + expect: + 1 == 2 + } +} diff --git a/src/test/java/FailingJUnit5Test.java b/src/test/java/FailingJUnit5Test.java new file mode 100644 index 0000000..d3baeba --- /dev/null +++ b/src/test/java/FailingJUnit5Test.java @@ -0,0 +1,18 @@ +import static org.junit.jupiter.api.Assertions.fail; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * Use this test to see whether maven executes JUnit5 tests or not. + */ +@Disabled +public class FailingJUnit5Test { + + @Test + @DisplayName("always fails") + void alwaysFails() { + fail("test fails as expected"); + } +} diff --git a/src/test/java/life/qbic/util/StringUtilTest.java b/src/test/java/life/qbic/util/StringUtilTest.java deleted file mode 100644 index 49fb9b3..0000000 --- a/src/test/java/life/qbic/util/StringUtilTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package life.qbic.util; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import org.junit.Test; - -public class StringUtilTest { - - @Test - public void endsWithIgnoreCase() { - // success with same case - assertTrue(StringUtil.endsWithIgnoreCase("thisissomerandomTESTBLA", "randomTESTBLA")); - assertTrue(StringUtil.endsWithIgnoreCase("thisissomerandomTESTBLA", "randomTESTBlA")); - assertTrue(StringUtil.endsWithIgnoreCase("thisissomerandomTESTBLA", "randomteSTBlA")); - // success with different case - assertTrue(StringUtil.endsWithIgnoreCase("thisissomerandomTESTBLA", "randomTESTbla")); - assertFalse(StringUtil.endsWithIgnoreCase("thisissomerandomTESTBLA", "ayyyyynope")); - } -} diff --git a/src/test/java/life/qbic/util/StringUtilsTest.java b/src/test/java/life/qbic/util/StringUtilsTest.java new file mode 100644 index 0000000..bea0bb7 --- /dev/null +++ b/src/test/java/life/qbic/util/StringUtilsTest.java @@ -0,0 +1,47 @@ +package life.qbic.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import life.qbic.qpostman.common.functions.EndsWithIgnoreCase; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + + +public class StringUtilsTest { + + @ParameterizedTest(name = "input = {0}") + @ValueSource(strings = {"test", "thisIs4L0n9_Text!"}) + @DisplayName("endsWithIgnoreCase is true for identical Strings") + void endsWithIgnoreCaseIsTrueForIdenticalStrings(String input) { + assertTrue(EndsWithIgnoreCase.endsWithIgnoreCase(input, input)); + } + + @ParameterizedTest(name = "input = {0} - {1}") + @CsvSource({ + "test, TEST", + "thisIs4L0n9_Text!, THISIS4L0N9_TEXT!"}) + @DisplayName("endsWithIgnoreCase is true for same Strings ignoring case") + void endsWithIgnoreCaseIsTrueForIdenticalStrings(String lowerCase, String upperCase) { + assertTrue(EndsWithIgnoreCase.endsWithIgnoreCase(lowerCase, upperCase)); + } + + @ParameterizedTest(name = "{0}\t{1} == {1}\t{0}") + @CsvSource({ + "test, TEST", + "thisIs4L0n9_Text!, THISIS4L0N9_TEXT!"}) + @DisplayName("endsWithIgnoreCase is symmetrical") + void endsWithIgnoreCaseIsSymmetrical(String one, String two) { + assertEquals(EndsWithIgnoreCase.endsWithIgnoreCase(one, two), EndsWithIgnoreCase.endsWithIgnoreCase(two, one)); + } + + @Test + @DisplayName("endsWithIgnoreCase is false for different Strings") + void endsWithIgnoreCaseIsFalseForDifferentStrings() { + assertFalse(EndsWithIgnoreCase.endsWithIgnoreCase("a", "b")); + } +}