component-hadoop/sample-app.yml at master · griddynamics/component-hadoop · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
application:
    configuration:
        input.dataset-url: "https://s3.amazonaws.com/qubell-hadoop-test/input/data.tar.gz"
        input.workflow-url: "https://s3.amazonaws.com/qubell-hadoop-test/workflow.tar.gz"
    interfaces:
        input:
            dataset-url: bind(workflow#input.dataset-url)
            workflow-url: bind(workflow#input.workflow-url)
        output:
            job-status: bind(workflow#result.job-status)
            data-url: bind(workflow#result.data-url)
            application-pic: "bind(metadata#output.application-pic)"
    bindings:
        - [ workflow, hadoop ]

    components:
        metadata:
            type: cobalt.common.Constants
            interfaces:
                output:
                    application-pic:
                        type: publish-signal(map<string, object>)
                        name: ""
            configuration:
                configuration.values:
                    output.application-pic:
                        large: "https://s3.amazonaws.com/qubell-images/datacloud.png"
                        small: "https://s3.amazonaws.com/qubell-images/datacloud_small.jpg"
                        small-height: 45

        hadoop:
            type: reference.Submodule
            configuration:
                __locator.application-id: "CDH Main"
            interfaces:
                Cloudera:
                    upload-data: receive-command(string archive-url => string data-dir, string data-url)
                    cleanup-data: receive-command(string data-dir)
                    run-workflow: receive-command(string archive-url, string data-dir => string status)

        workflow:
            type: workflow.Instance
            interfaces:
                input:
                    dataset-url:
                        type: configuration(string)
                        name: Data archive
                    workflow-url:
                        type: configuration(string)
                        name: Workflow archive
                result:
                    data-url:
                        type: publish-signal(string)
                        name: View results
                    job-status:
                        type: publish-signal(string)
                        name: Job result
                    data-dir:
                        type: publish-signal(string)
                        name: Data directory
                hadoop:
                    upload-data: send-command(string archive-url => string data-dir, string data-url)
                    cleanup-data: send-command(string data-dir)
                    run-workflow: send-command(string archive-url, string data-dir => string status)

            required: [ hadoop ]
            configuration:
                configuration.propagateStatus: []
                configuration.triggers: {}
                configuration.workflows:
                    launch:
                        steps:
                            - upload-data:
                                action: hadoop.upload-data
                                parameters:
                                    commandCallTimeout: 600
                                    archive-url: "{$.dataset-url}"
                                output:
                                    data-dir: data-dir
                                    data-url: data-url
                            - run-workflow:
                                action: hadoop.run-workflow
                                precedingPhases: [ upload-data ]
                                parameters:
                                    commandCallTimeout: 3600
                                    archive-url: "{$.workflow-url}"
                                    data-dir: "{$.data-dir}"
                                output:
                                    status: status
                        return:
                            data-url:
                                value: "{$.data-url}"
                            job-status:
                                value: "{$.status}"
                            data-dir:
                                value: "{$.data-dir}"
                    destroy:
                        steps:
                            - cleanup-data:
                                action: hadoop.cleanup-data
                                parameters:
                                    commandCallTimeout: 600
                                    data-dir: "{$.data-dir}"