1#!/usr/bin/env cwl-runner
2### Full AirNowProcessing Pipeline (with shapefiles on local file system)
3# Copyright (c) 2021. Harvard University
4#
5# Developed by Research Software Engineering,
6# Faculty of Arts and Sciences, Research Computing (FAS RC)
7# Author: Michael A Bouzinier
8#
9# Licensed under the Apache License, Version 2.0 (the "License");
10# you may not use this file except in compliance with the License.
11# You may obtain a copy of the License at
12#
13# http://www.apache.org/licenses/LICENSE-2.0
14#
15# Unless required by applicable law or agreed to in writing, software
16# distributed under the License is distributed on an "AS IS" BASIS,
17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18# See the License for the specific language governing permissions and
19# limitations under the License.
20#
21
22cwlVersion: v1.2
23class: Workflow
24
25requirements:
26 SubworkflowFeatureRequirement: {}
27 StepInputExpressionRequirement: {}
28 InlineJavascriptRequirement: {}
29
30doc: |
31 This workflow downloads AirNow data from the government
32 servers, introspects it to infer the database schema
33 and ingests the data into the database
34
35 Starting with version 0.5.0, the workflow initializes the database
36 (via initcoredb.cwl) before ingesting the data, hence it can be run
37 against a pristine, empty database. Earlier versions required the
38 database to be initialized beforehand (e.g., by running another
39 pipeline first).
40
41inputs:
42 proxy:
43 type: string?
44 default: ""
45 doc: HTTP/HTTPS Proxy if required
46 api-key:
47 type: string
48 doc: API key for AirNow
49 database:
50 type: File
51 doc: Path to database connection file, usually database.ini
52 connection_name:
53 type: string
54 doc: The name of the section in the database.ini file
55 from:
56 type: string
57 doc: Start date for downolading, in YYYY-MM-DD format
58 to:
59 type: string
60 doc: End date for downolading, in YYYY-MM-DD format
61 parameter_code:
62 type: string
63 doc: |
64 Parameter code. Either a numeric code (e.g. 88101, 44201)
65 or symbolic name (e.g. PM25, NO2).
66 See more: [AQS Code List](https://www.epa.gov/aqs/aqs-code-list)
67 table:
68 doc: Name of the table to be created in the database
69 type: string
70 shapes:
71 type: File[]
72 secondaryFiles:
73 - "^.dbf"
74 - "^.shx"
75 - "^.prj"
76 - "^.cpg"
77
78steps:
79 download:
80 run: download_airnow.cwl
81 in:
82 api-key: api-key
83 shapes: shapes
84 from: from
85 to: to
86 table: table
87 parameter_code: parameter_code
88 proxy: proxy
89 out: [log, data]
90
91 introspect:
92 run: introspect.cwl
93 in:
94 depends_on: download/log
95 input: download/data
96 table: table
97 output:
98 valueFrom: epa.yaml
99 out: [log, model]
100
101 initdb:
102 run: initcoredb.cwl
103 doc: Ensure that database utilities are at their latest version
104 in:
105 database: database
106 connection_name: connection_name
107 out:
108 - log
109 - err
110
111 ingest:
112 run: ingest.cwl
113 doc: Uploads data into the database
114 in:
115 depends_on: initdb/log
116 registry: introspect/model
117 domain:
118 valueFrom: "epa"
119 table: table
120 input: download/data
121 database: database
122 connection_name: connection_name
123 out: [log]
124
125 index:
126 run: index.cwl
127 in:
128 depends_on: ingest/log
129 registry: introspect/model
130 domain:
131 valueFrom: "epa"
132 table: table
133 database: database
134 connection_name: connection_name
135 out: [log]
136
137outputs:
138 download_log:
139 type: File
140 outputSource: download/log
141 ingest_log:
142 type: File
143 outputSource: ingest/log
144 index_log:
145 type: File
146 outputSource: index/log
147 download_data:
148 type: File
149 outputSource: download/data
150 model:
151 type: File
152 outputSource: introspect/model