airnow.cwl

  1#!/usr/bin/env cwl-runner
  2### Full EPA AirNow Processing Pipeline (including downloading shapefiles)
  3#  Copyright (c) 2021. Harvard University
  4#
  5#  Developed by Research Software Engineering,
  6#  Faculty of Arts and Sciences, Research Computing (FAS RC)
  7#  Author: Michael A Bouzinier
  8#
  9#  Licensed under the Apache License, Version 2.0 (the "License");
 10#  you may not use this file except in compliance with the License.
 11#  You may obtain a copy of the License at
 12#
 13#         http://www.apache.org/licenses/LICENSE-2.0
 14#
 15#  Unless required by applicable law or agreed to in writing, software
 16#  distributed under the License is distributed on an "AS IS" BASIS,
 17#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 18#  See the License for the specific language governing permissions and
 19#  limitations under the License.
 20#
 21
 22cwlVersion: v1.2
 23class: Workflow
 24
 25requirements:
 26  SubworkflowFeatureRequirement: {}
 27  StepInputExpressionRequirement: {}
 28  InlineJavascriptRequirement: {}
 29
 30doc: |
 31  This workflow downloads AirNow data from the government
 32  servers, introspects it to infer the database schema
 33  and ingests the data into the database
 34
 35  Starting with version 0.5.0, the workflow initializes the database
 36  (via initcoredb.cwl) before ingesting the data, hence it can be run
 37  against a pristine, empty database. Earlier versions required the
 38  database to be initialized beforehand (e.g., by running another
 39  pipeline first).
 40
 41  Example run:
 42  ```shell
 43  cwl-runner airnow.cwl sample_airnow.yml
 44  ```
 45
 46  See [sample_airnow.yml](sample_airnow.md)
 47
 48  Or
 49
 50  ```shell
 51  cwl-runner --parallel /opt/airflow/project/epa/src/cwl/airnow.cwl --database /opt/airflow/project/database.ini --connection_name nsaph2 --proxy $HTTP_PROXY  --api-key XXXXXXXX-YYYY-ZZZZ-XXXX-YYYYYYYYY --from 2022-01-01 --to 2022-08-31 --parameter_code pm25 --table airnow_pm25_2022
 52  ```
 53
 54inputs:
 55  proxy:
 56    type: string?
 57    default: ""
 58    doc: HTTP/HTTPS Proxy if required
 59  api-key:
 60    type: string
 61    doc: API key for AirNow
 62  database:
 63    type: File
 64    doc: Path to database connection file, usually database.ini
 65  connection_name:
 66    type: string
 67    doc: The name of the section in the database.ini file
 68  from:
 69    type: string
 70    doc: Start date for downolading, in YYYY-MM-DD format
 71  to:
 72    type: string
 73    doc: End date for downolading, in YYYY-MM-DD format
 74  parameter_code:
 75    type: string
 76    doc: |
 77      Parameter code. Either a numeric code (e.g. 88101, 44201)
 78      or symbolic name (e.g. PM25, NO2).
 79      See more: [AQS Code List](https://www.epa.gov/aqs/aqs-code-list)
 80  table:
 81    doc: Name of the table to be created in the database
 82    type: string
 83  year:
 84    type: int
 85
 86steps:
 87  get_shapes:
 88    run: get_shapes.cwl
 89    doc: |
 90      This step downloads Shape files from a given collection (TIGER/Line or GENZ) 
 91      and a geography (ZCTA or Counties) from the US Census website,
 92      for a given year or for the closest one.
 93
 94    in:
 95      year:
 96        valueFrom: $(String(inputs.yy))
 97      yy: year
 98      geo:
 99        valueFrom: "all"
100      proxy: proxy
101    out: [shape_files]
102
103  download:
104    run: download_airnow.cwl
105    in:
106      api-key: api-key
107      shapes: get_shapes/shape_files
108      from: from
109      to: to
110      table: table
111      parameter_code: parameter_code
112      proxy: proxy
113    out: [log, data]
114
115  introspect:
116    run: introspect.cwl
117    in:
118      depends_on: download/log
119      input: download/data
120      table: table
121      output:
122        valueFrom: epa.yaml
123    out: [log, model]
124
125  initdb:
126    run: initcoredb.cwl
127    doc: Ensure that database utilities are at their latest version
128    in:
129      database: database
130      connection_name: connection_name
131    out:
132      - log
133      - err
134
135  ingest:
136    run: ingest.cwl
137    doc: Uploads data into the database
138    in:
139      depends_on: initdb/log
140      registry: introspect/model
141      domain:
142        valueFrom: "epa"
143      table: table
144      input: download/data
145      database: database
146      connection_name: connection_name
147    out: [log]
148
149  index:
150    run: index.cwl
151    in:
152      depends_on: ingest/log
153      registry: introspect/model
154      domain:
155        valueFrom: "epa"
156      table: table
157      database: database
158      connection_name: connection_name
159    out: [log]
160
161  vacuum:
162    run: vacuum.cwl
163    in:
164      depends_on: index/log
165      registry: introspect/model
166      domain:
167        valueFrom: "epa"
168      table: table
169      database: database
170      connection_name: connection_name
171    out: [log]
172
173
174outputs:
175  shapes_data:
176    type: File[]
177    outputSource: get_shapes/shape_files
178  download_log:
179    type: File
180    outputSource: download/log
181  ingest_log:
182    type: File
183    outputSource: ingest/log
184  index_log:
185    type: File
186    outputSource: index/log
187  vacuum_log:
188    type: File
189    outputSource: vacuum/log
190  download_data:
191    type: File
192    outputSource: download/data
193  model:
194    type: File
195    outputSource: introspect/model