diff --git a/_data/navigation.yml b/_data/navigation.yml index 3e7e2321..ffb6bb55 100644 --- a/_data/navigation.yml +++ b/_data/navigation.yml @@ -17,36 +17,6 @@ items: - url: /integrate/ title: Integration items: - - url: /integrate/storage/ - title: Storage - items: - - url: /integrate/storage/php-client/ - title: PHP client library - - - url: /integrate/storage/r-client/ - title: R client library - - - url: /integrate/storage/python-client/ - title: Python client library - - - url: /integrate/storage/docker-cli-client/ - title: Docker CLI client - - - url: /integrate/storage/api/ - title: Using API - items: - - url: /integrate/storage/api/configurations/ - title: Configurations API - - - url: /integrate/storage/api/importer/ - title: Storage API Importer - - - url: /integrate/storage/api/import-export/ - title: Manually importing and exporting data - - - url: /integrate/storage/api/tde-exporter/ - title: TDE Exporter - - url: /integrate/jobs/ title: Component Jobs diff --git a/index.md b/index.md index f6ed3864..f8b18c8f 100644 --- a/index.md +++ b/index.md @@ -20,7 +20,7 @@ such as Extractors, Storage or Writers, that are orchestrated together through ( In this documentation, we will show you how to - [**Integrate Keboola with other systems**](/integrate/). - - Use Keboola just to exchange data (using the [Storage API](/integrate/storage/)). + - Use Keboola just to exchange data (using the [Storage API](https://help.keboola.com/storage/api/)). - Use Keboola as a [data-handling backbone](/overview/api/) for your product. - Wrap Keboola in your own UI for your customers. - Control whole data processing pipeline within Keboola from the [outside](/integrate/). diff --git a/integrate/index.md b/integrate/index.md index 62a57dfa..689c8c9e 100644 --- a/integrate/index.md +++ b/integrate/index.md @@ -11,9 +11,9 @@ We do not cheat or have any advantage over other developers; our UI and other co As a result, it is very easy to, for example, write custom scripts to bootstrap a project, or do something that our UI does not offer. Let's have a look into this! -One of the very important components is [Storage](/integrate/storage/), which not only stores all data in a +One of the very important components is [Storage](https://help.keboola.com/storage/api/), which not only stores all data in a project, but also provides additional functions such as managing other components and their configurations. -When you are integrating your systems with Keboola, **chances are that you want to start with [Storage](/integrate/storage/)**. +When you are integrating your systems with Keboola, **chances are that you want to start with [Storage](https://help.keboola.com/storage/api/)**. {% comment %} diff --git a/integrate/storage/api/async-import-handling.svg b/integrate/storage/api/async-import-handling.svg deleted file mode 100644 index 777e9337..00000000 --- a/integrate/storage/api/async-import-handling.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/integrate/storage/api/configurations.md b/integrate/storage/api/configurations.md index 8e4fd494..8733bc82 100644 --- a/integrate/storage/api/configurations.md +++ b/integrate/storage/api/configurations.md @@ -1,525 +1,5 @@ ---- -title: Component Configurations API -permalink: /integrate/storage/api/configurations/ ---- - -* TOC -{:toc} - -[Configurations](https://help.keboola.com/storage/configurations/) are an important part of a Keboola project. Most operations are -available in the UI. Use the API if you want to manipulate the configurations programmatically. - -Configurations represent component **instances** in a project. Each Keboola component has different configuration -options and requirements, which must be respected. As such, Keboola configurations provide a general framework for configuring -components, while the specific implementation details are left to the components themselves. - -When working with the [Component Configurations API](https://api.keboola.com/?service=storage#tag--Component-Configurations), -you need to know the `componentId` of the component being configured. -You can see a list of public components in [the Developer Portal](https://components.keboola.com/components), or you can get -a list of all available components with the [API index call](https://api.keboola.com/?service=storage#get-/v2/storage). -See our [example](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb). - -It will give you something like this: - -{% highlight json %} -{ - "host": "4edece0b0052", - "api": "storage", - "version": "v2", - "revision": "21fb56a0f6d61a307f350247a45950b1e4049625", - "documentation": "https://connection.keboola.com/api/storage/doc.json", - "components": [ - { - "id": "keboola.ex-aws-s3", - "type": "extractor", - "name": "AWS S3", - "description": "AWS Simple Storage Service", - "longDescription": "Download ... from AWS S3 and upload them to Storage.", - "version": 23, - "hasUI": false, - "hasRun": false, - "ico32": "https://ui.keboola-assets.com/.../keboola.ex-aws-s3/32/20.png", - "ico64": "https://ui.keboola-assets.com/.../keboola.ex-aws-s3/64/20.png", - "data": { - "definition": { - "type": "aws-ecr", - "uri": "147946154733.../keboola.ex-aws-s3", - "tag": "v3.0.0", - "repository": { - "region": "us-east-1" - } - }, - "vendor": { - "contact": [ - "Keboola", - "Křižíkova 488/115\n186 00 Prague 8\nCzech Republic", - "support@keboola.com" - ], - "licenseUrl": "https://github.com/keboola/aws-s3-extractor/blob/master/LICENSE" - }, - "configuration_format": "json", - "network": "bridge", - "memory": "512m", - "forward_token": false, - "forward_token_details": false, - "default_bucket": true, - "default_bucket_stage": "in", - "staging_storage": { - "input": "local" - } - }, - "flags": [ - "genericDockerUI", - "genericDockerUI-processors", - "appInfo.dataIn" - ], - "configurationSchema": {}, - "emptyConfiguration": {}, - "uiOptions": {}, - "configurationDescription": null, - "documentationUrl": "https://help.keboola.com/extractors/other/aws-s3/" - } - ], - "services": [...], - "urlTemplates": {...} -} -{% endhighlight %} - -From here, you can see all available information about a particular component. In the following examples, we -will use `keboola.ex-aws-s3` --- the AWS S3 extractor. - -## Configuration Structure -Component configurations are largely dependent on the actual component being configured. This makes creating configurations manually -a bit tricky. Rather than starting from scratch, we recommend creating a configuration through the UI and then modifying it when you understand it. - -### Inspecting Configuration -To obtain an existing configuration, you can either use the list of configurations above or -the [Configuration Detail](https://api.keboola.com/?service=storage#get-/v2/storage/branch/-branchId-/components/-componentId-/configs/-configurationId-) -API call. See an [example](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb) for obtaining a -configuration of the `keboola.ex-aws-s3` component. You will receive a response similar to this: - -{% highlight json %} -{ - "id": "364479526", - "name": "test", - "description": "", - "created": "2018-03-08T14:54:19+0100", - "creatorToken": { - "id": 27865, - "description": "ondrej.popelka@keboola.com" - }, - "version": 5, - "changeDescription": "Table first table edited", - "isDeleted": false, - "configuration": { - "parameters": { - "accessKeyId": "AKIAIBZYEEXQILP46FCA", - "#secretAccessKey": "KBC::ComponentProjectEncrypted==p5gvUw4RSGiVJjT2ayVORpqS7yiKhExi7NnQECntVm8haHaHtFNVDMT8X8b+htnixpXhPIQ9yV+ETrvr+hNeYfh+Ex+UpC//QPWnLcEOC8XOLgmQN8BNgRGSERWUziK0" - } - }, - "rowsSortOrder": [], - "rows": [ - { - "id": "364481153", - "name": "first table", - "description": "", - "configuration": { - "parameters": { - "bucket": "travis-php-db-import-tests-s3filesbucket-vm9zhtm5jd7s", - "key": "tw_accounts.csv", - "saveAs": "first-table", - "includeSubfolders": false, - "newFilesOnly": true - }, - "processors": { - "after": [ - { - "definition": { - "component": "keboola.processor-move-files" - }, - "parameters": { - "direction": "tables", - "addCsvSuffix": true - } - }, - { - "definition": { - "component": "keboola.processor-create-manifest" - }, - "parameters": { - "delimiter": ",", - "enclosure": "\"", - "incremental": false, - "primary_key": [], - "columns": [], - "columns_from": "header" - } - }, - { - "definition": { - "component": "keboola.processor-skip-lines" - }, - "parameters": { - "lines": 1 - } - } - ] - } - }, - "isDisabled": false, - "version": 3, - "created": "2018-03-08T14:58:33+0100", - "creatorToken": { - "id": 27865, - "description": "ondrej.popelka@keboola.com" - }, - "changeDescription": "Table first table edited", - "state": { - "lastDownloadedFileTimestamp": "1511176959", - "processedFilesInLastTimestampSecond": [ - "tw_accounts.csv" - ] - } - } - ], - "state": {}, - "currentVersion": { - "created": "2018-03-08T23:27:37+0100", - "creatorToken": { - "id": 27865, - "description": "ondrej.popelka@keboola.com" - }, - "changeDescription": "Table first table edited" - } -} -{% endhighlight %} - -The actual component configuration is split into three parts: - -- `configuration` node, containing an arbitrary component configuration -- `state` node, containing a component [state file](https://help.keboola.com/extend/common-interface/config-file/#state-file) -- `rows` node, containing iterations of `configuration` and `state` - -The important part is the ID of the configuration you want to work with. In the following examples, we will use -`364479526`. - -### Configuration -The `configuration` node maps to the [configuration file](https://help.keboola.com/extend/common-interface/config-file/#configuration-file-structure). -It can contain the `storage`, `parameters`, `processors` and `authorization` child nodes (the `image_parameters` and `action` nodes found in the config file -are injected at runtime and are not stored in the configuration). The `authorization` node is set in the configuration only when -[credentials injection](https://help.keboola.com/extend/common-interface/oauth/#credentials-injection) should be used, otherwise it is also set during the runtime. -The `processors` node defines the [processors and their configuration](https://help.keboola.com/extend/component/processors/). -The most common sub-nodes stored in the `configuration` node are therefore `parameters` (containing an arbitrary component configuration) -and `storage` (containing [input](https://help.keboola.com/extend/component/tutorial/input-mapping/) and [output mapping](https://help.keboola.com/extend/component/tutorial/output-mapping/)). -Both are transferred to the -configuration file without modification; that means that the [`storage` configuration](https://help.keboola.com/extend/common-interface/config-file/#configuration-file-structure) -is directly usable in the `configuration` node. The `parameters` node is fully dependent on the component and has no universal specification or rules. - -In the above example, the `configuration` node contains the following: - -{% highlight json %} -"parameters": { - "accessKeyId": "AKIAIBZYEEXQILP46FCA", - "#secretAccessKey": "KBC::ComponentProjectEncrypted==p5gvUw4RSGiVJjT2ayVORpqS7yiKhExi7NnQECntVm8haHaHtFNVDMT8X8b+htnixpXhPIQ9yV+ETrvr+hNeYfh+Ex+UpC//QPWnLcEOC8XOLgmQN8BNgRGSERWUziK0" -} -{% endhighlight %} - -That means that the component is not using input mapping nor output mapping. The allowed contents of `parameters` are described -in the [AWS S3 extractor code documentation](https://github.com/keboola/aws-s3-extractor#configuration-options). - -### Configuration Rows -The `rows` node contains iterations of the configuration. The interpretation of configuration rows is again dependent on the -component implementation. In the presented case of the `keboola.ex-aws-s3` component, each row corresponds to a single extracted table. -When `rows` node is non-empty, the component behavior is slightly modified. It behaves as if it were executed as many times as -there are rows. For each row, the `configuration` node from `root` and the `configuration` node from `rows` are merged, with -the latter overwriting the former in the case of conflict. - -Given the above configuration, the **effective configuration** passed to the component -[configuration file](https://help.keboola.com/extend/common-interface/config-file/#configuration-file-structure) will be as follows: - -{% highlight json %} -{ - "parameters": { - "accessKeyId": "AKIAIBZYEEXQILP46FCA", - "#secretAccessKey": "KBC::ComponentProjectEncrypted==p5gvUw4RSGiVJjT2ayVORpqS7yiKhExi7NnQECntVm8haHaHtFNVDMT8X8b+htnixpXhPIQ9yV+ETrvr+hNeYfh+Ex+UpC//QPWnLcEOC8XOLgmQN8BNgRGSERWUziK0" - "bucket": "travis-php-db-import-tests-s3filesbucket-vm9zhtm5jd7s", - "key": "tw_accounts.csv", - "saveAs": "first-table", - "includeSubfolders": false, - "newFilesOnly": true - } -} -{% endhighlight %} - -The first two parameters (`accessKeyId` and `#secretAccessKey`) are taken from the root `configuration`, the other -parameters are taken from the first rows' `configuration`. The `processors` node is never passed to the configuration file. -With the above configuration, the component will be executed only once, because there is one row. If there are no rows, the -component will still be executed once. If there were two rows, the component would be executed twice. - -If the component is executed more than once, the operations are executed in the following order: - -- input mapping for the first row -- run with the first row configuration (merged with root configuration) -- output mapping for the first row -- input mapping for the second row -- run with the second row configuration (merged with root configuration) -- output mapping for the second row - -All of these are executed in a single [job](/integrate/jobs/). However, even though multiple rows are executed in a single -job, the actual executions are still completely isolated. I.e., there is no way to share anything between the rows -(apart from the common `configuration`). It also means that the outputs of the first row are available in the Keboola project before -the second row starts, and the inputs for the second row are read only after the first row finishes processing. - -What is considered 'first' and 'second' -- i.e. the order of rows -- is defined by the order of items in the `rows` array. -See [below](#modifying-a-configuration) for an example of modifying the row order. - -Theoretically, configuration rows are supported for every component as long as the effective configuration matches what -the component expects. Configuration rows can be used to split the configuration into a common part (typically credentials) and an -iterable part which is repeated many times. Keep in mind that configurations heavily modified through the API might **not be supported -in the UI**. - -### State -The `state` node contains the content of the [state file](https://help.keboola.com/extend/common-interface/config-file/#state-file). The -`state` is read from the state file and then supplied to the state file on the next run. In the above configuration, -the state is: - -{% highlight json %} -{ - "lastDownloadedFileTimestamp": "1511176959", - "processedFilesInLastTimestampSecond": [ - "tw_accounts.csv" - ] -} -{% endhighlight %} - -`State` is considered an internal property of a component and you should avoid modifying it. The only reasonable modification of -`state` is to delete it -- in that case, the configuration will run as if it were run for the first time. To delete the `state`, set it to `{}`. -If configuration rows are used, then the `state` is stored separately for each row and the `state` node in configuration root is -not used. - -## Working with Configurations -Here, the most common operations done with configurations are described in examples. Feel free to go through the -[API reference](https://api.keboola.com/?service=storage#tag--Component-Configurations) for a full authoritative list of configuration features. - -### List Configurations -To obtain configuration details, use the [List Configs call](https://api.keboola.com/?service=storage#get-/v2/storage/branch/-branchId-/components/-componentId-/configs), -which will return all the configuration details. This means - -- the configuration itself (`configuration`) --- [section on configuration](#modifying-a-configuration) follows; -- configuration rows (`rows`) --- additional data of the configuration; and -- configuration state (`state`) --- [component state](https://help.keboola.com/extend/common-interface/config-file/#state-file). - -Please note that the contents -of the `configuration`, `rows` and `state` sections depend purely on the component itself. See an [example](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb). - -A sample result for the AWS S3 extractor looks like this: - -{% highlight json %} -[ - { - "id": "364479526", - "name": "test", - "description": "", - "created": "2018-03-08T14:54:19+0100", - "creatorToken": { - "id": 27865, - "description": "ondrej.popelka@keboola.com" - }, - "version": 4, - "changeDescription": "Table first table edited", - "isDeleted": false, - "configuration": { - "parameters": { - "accessKeyId": "AKIAIBZYEEXQILP46FCA", - "#secretAccessKey": "KBC::ComponentProjectEncrypted==p5gvUw4RSGiVJjT2ayVORpqS7yiKhExi7NnQECntVm8haHaHtFNVDMT8X8b+htnixpXhPIQ9yV+ETrvr+hNeYfh+Ex+UpC//QPWnLcEOC8XOLgmQN8BNgRGSERWUziK0" - } - }, - "rowsSortOrder": [], - "rows": [ - { - "id": "364481153", - "name": "first table", - "description": "", - "configuration": {...}, - "isDisabled": false, - "version": 2, - "created": "2018-03-08T14:58:33+0100", - "creatorToken": { - "id": 27865, - "description": "ondrej.popelka@keboola.com" - }, - "changeDescription": "Table first table edited", - "state": {} - } - ], - "state": {}, - "currentVersion": { - "created": "2018-03-08T15:21:28+0100", - "creatorToken": { - "id": 27865, - "description": "ondrej.popelka@keboola.com" - }, - "changeDescription": "Table first table edited" - } - } -] -{% endhighlight %} - -### Modifying Configuration -**Note: Configurations modified through the API might not be editable in the Keboola UI.** They can be run or used in an orchestration without any problems. - -Modifying a configuration means that a new version of that configuration is created. -For modifying a configuration, use the -[Update Configuration](https://api.keboola.com/?service=storage#put-/v2/storage/branch/-branchId-/components/-componentId-/configs/-configurationId-) API call. -See an [example](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb) in which the -configuration is modified to the following to set new credentials: - -{% highlight json %} -{ - "parameters": { - "accessKeyId": "a", - "#secretAccessKey": "b" - } -} -{% endhighlight %} - -Notice that the configuration must be sent in the form field `configuration` as the endpoint does not accept pure JSON (yet). -Take great care to pass **only the contents** of the `configuration` node as in the above example. The configuration **must not be wrapped** in the -`configuration` node, otherwise the component will not -receive the configuration it expects. Also take care to properly escape the JSON using [URL encoding](https://en.wikipedia.org/wiki/Percent-encoding), -otherwise it may be misinterpreted. The raw HTTP request should look similar to this: - - curl --request PUT \ - --url https://connection.keboola.com/v2/storage/components/keboola.ex-aws-s3/configs/364479526 \ - --header "Content-Type: application/json" \ - --header 'X-StorageAPI-Token: {{token}}' \ - --data-binary "{ - \"configuration\": { - \"parameters\": { - \"accessKeyId\": \"a\", - \"#secretAccessKey\": \"b\" - } - } - }" - -Also note that the entire configuration must be always sent, there is no way to patch only part of it. -The same way the `configuration` is modified, other properties can be modified too. For example, you may want to -reset `state` by setting it to `{}`, or you can change the order of the configuration rows by setting the `rowsSortOrder` property. -The `rowsSortOrder` is an array of row ids -- see an [example](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb) (Set Row order of S3 extractor) -for the exact example request. - -### Modifying Configuration Row -Very similar to modifying a configuration, modifying a configuration **row** means that a new version of -the **entire configuration** is created. For modifying a configuration row, use the -[Update Row](https://api.keboola.com/?service=storage#put-/v2/storage/branch/-branchId-/components/-componentId-/configs/-configurationId-/rows/-rowId-) API call. - -See an [example](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb) in which the -configuration row is modified to: - -{% highlight json %} -{ - "parameters": { - "bucket": "some-bucket", - "key": "sample.csv", - "includeSubfolders": false, - "newFilesOnly": true - } -} -{% endhighlight %} - -The rules for updating a configuration row are the same as for [updating a configuration](#modifying-a-configuration). Also note that -a configuration row is never evaluated alone, it is always merged with the root `configuration`. If the same properties are defined -in the root `configuration` and row `configuration`, the values from the row are used. There is also an -[example](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb) of how to reset the row -state by setting `state` to `{}`. - -### Configuration Versions -When you [update a configuration](https://api.keboola.com/?service=storage#put-/v2/storage/branch/-branchId-/components/-componentId-/configs/-configurationId-), -a new configuration version is actually created. In the above calls, only the last (active/published) configuration -is returned. To obtain a list of all recorded versions, use the -[List Versions API call](https://api.keboola.com/?service=storage#get-/v2/storage/branch/-branchId-/components/-componentId-/configs/-configurationId-/versions). -See this [example](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb) -which would give you an output similar to the one below: - -{% highlight json %} -[ - { - "version": 4, - "created": "2018-03-08T15:21:28+0100", - "creatorToken": { - "id": 27865, - "description": "ondrej.popelka@keboola.com" - }, - "changeDescription": "Table first table edited", - "isDeleted": false, - "name": "test", - "description": "" - }, - { - "version": 3, - "created": "2018-03-08T14:58:33+0100", - "creatorToken": { - "id": 27865, - "description": "ondrej.popelka@keboola.com" - }, - "changeDescription": "Table first table added", - "isDeleted": false, - "name": "test", - "description": "" - }, - { - "version": 2, - "created": "2018-03-08T14:55:50+0100", - "creatorToken": { - "id": 27865, - "description": "ondrej.popelka@keboola.com" - }, - "changeDescription": "AWS Credentials edited", - "isDeleted": false, - "name": "test", - "description": "" - }, - { - "version": 1, - "created": "2018-03-08T14:54:19+0100", - "creatorToken": { - "id": 27865, - "description": "ondrej.popelka@keboola.com" - }, - "changeDescription": "", - "isDeleted": false, - "name": "test", - "description": "" - } -] -{% endhighlight %} - -The field `version` represents the `version_id` in the following API example. - -### Rollback Configuration -After choosing a particular version, you can revert to that version by -[rolling back](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/components/-componentId-/configs/-configurationId-/versions/-versionId-/rollback), -i.e., making a new version identical to the chosen one. See an [example](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D#2050856a-66b3-4120-9552-d1278a96621e) -of how to rollback the configuration `364479526` of the `keboola.ex-aws-s3` component to version `3`. - -It will create a new version of the configuration and return the ID of the version: -{% highlight json %} -{ - "version": "26" -} -{% endhighlight %} - -### Creating Configuration Copy -After choosing a particular version, you can create a new independent -[configuration copy](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/components/-componentId-/configs/-configurationId-/versions/-versionId-/create) -of it. See an [example](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb) -of how to create a new configuration called `test-copy` from version `3` of the `364479526` configuration -for the `keboola.ex-aws-s3` component. - -It will return the ID of the newly created configuration: -{% highlight json %} -{ - "id": "364494012" -} -{% endhighlight %} - +--- +title: Component Configurations API +permalink: /integrate/storage/api/configurations/ +redirect_to: https://help.keboola.com/storage/api/configurations/ +--- diff --git a/integrate/storage/api/import-export.md b/integrate/storage/api/import-export.md index 8d3dfdd3..8182d24e 100644 --- a/integrate/storage/api/import-export.md +++ b/integrate/storage/api/import-export.md @@ -1,270 +1,5 @@ ---- -title: Manually Importing and Exporting Data -permalink: /integrate/storage/api/import-export/ ---- - -* TOC -{:toc} - -## Working with Data -Keboola Table Storage (Tables) and Keboola File Storage (File Uploads) are heavily connected together. -Keboola File Storage is technically a layer on top of the Amazon S3 service, and Keboola Table -Storage is a layer on top of a [database backend](https://help.keboola.com/storage/#backends). - -To upload a table, take the following steps: - -- Request a [file upload](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/files/prepare) from -Keboola File Storage. You will be given a destination for the uploaded file on an S3 server. -- Upload the file there. When the upload is finished, the data file will be available in the *File Uploads* section. -- Initiate an [asynchronous table import](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/tables/-id-/import-async) -from the uploaded file (use it as the `dataFileId` parameter) into the destination table. -The import is asynchronous, so the request only creates a job and you need to poll for its results. -The imported files must conform to the [RFC4180 Specification](https://tools.ietf.org/html/rfc4180). - -{: .image-popup} -![Schema of file upload process](/integrate/storage/api/async-import-handling.svg) - -Exporting a table from Storage is analogous to its importing. First, data is [asynchronously -exported](https://keboola.docs.apiary.io/#reference/tables/unload-data-asynchronously/asynchronous-export) from -Table Storage into File Uploads. Then you can request to [download -the file](https://api.keboola.com/?service=storage#get-/v2/storage/branch/-branchId-/files/-fileId-), which will give you -access to an S3 server for the actual file download. - -### Manually Uploading a File -To upload a file to Keboola File Storage, follow the instructions outlined in the -[API documentation](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/files/prepare). -First create a file resource; to create a new file called -[`new-file.csv`](/integrate/storage/new-table.csv) with `52` bytes, call: - -{% highlight bash %} -curl --request POST --header "Content-Type: application/json" --header "X-StorageApi-Token:storage-token" --data-binary "{ \"name\": \"new-file.csv\", \"sizeBytes\": 52, \"federationToken\": 1 }" https://connection.keboola.com/v2/storage/files/prepare -{% endhighlight %} - -Which will return a response similar to this: - -{% highlight json %} -{ - "id": 192726698, - "created": "2016-06-22T10:44:35+0200", - "isPublic": false, - "isSliced": false, - "isEncrypted": false, - "name": "new_file2.csv", - "url": "https://s3.amazonaws.com/kbc-sapi-files/exp-15/1134/files/2016/06/22/192726697.new_file2?X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAJ2N244XSWYVVYVLQ%2F20160622%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20160622T084435Z&X-Amz-SignedHeaders=host&X-Amz-Expires=3600&X-Amz-Signature=86136cced74cdf919953cde9e2a0b837bd0b8f147aa6b7b30c2febde3b92d83d", - "region": "us-east-1", - "sizeBytes": 52, - "tags": [], - "maxAgeDays": 15, - "runId": null, - "runIds": [], - "creatorToken": { - "id": 53044, - "description": "ondrej.popelka@keboola.com" - }, - "uploadParams": { - "key": "exp-15/1134/files/2016/06/22/192726697.new_file2.csv", - "bucket": "kbc-sapi-files", - "acl": "private", - "credentials": { - "AccessKeyId": "ASI...H7Q", - "SecretAccessKey": "QbO...7qu", - "SessionToken": "Ago...bsF", - "Expiration": "2016-06-22T20:44:35+00:00" - } - } -} -{% endhighlight %} - -The important parts are: `id` of the file, which will be needed later, the `uploadParams.credentials` node, -which gives you credentials to AWS S3 to upload your file, and -the `key` and `bucket` nodes, which define the target S3 destination as *s3://`bucket`/`key`*. -To upload the files to S3, you need an S3 client. There are a large number of clients available: -for example, use the -[S3 AWS command line client](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html). -Before using it, [pass the credentials](https://docs.aws.amazon.com/cli/latest/topic/config-vars.html#credentials) -by executing, for instance, the following commands - -on *nix systems: -{% highlight bash %} -export AWS_ACCESS_KEY_ID=ASI...H7Q -export AWS_SECRET_ACCESS_KEY=QbO...7qu -export AWS_SESSION_TOKEN=Ago...wU= -{% endhighlight %} - -or on Windows: -{% highlight bash %} -SET AWS_ACCESS_KEY_ID=ASI...H7Q -SET AWS_SECRET_ACCESS_KEY=QbO...7qu -SET AWS_SESSION_TOKEN=Ago...bsF -{% endhighlight %} - -Then you can actually upload the `new-table.csv` file by executing the AWS S3 CLI [cp command](https://docs.aws.amazon.com/cli/latest/reference/s3/cp.html): -{% highlight bash %} -aws s3 cp new-table.csv s3://kbc-sapi-files/exp-15/1134/files/2016/06/22/192726697.new_file2.csv -{% endhighlight %} - -After that, import the file into Table Storage, by calling either -[Create Table API call](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/buckets/-id-/tables-async) -(for a new table) or -[Load Data API call](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/tables/-id-/import-async) -(for an existing table). - -{% highlight bash %} -curl --request POST --header "Content-Type: application/json" --header "X-StorageApi-Token:storage-token" --data-binary "{ \"dataFileId\": 192726698, \"name\": \"new-table\" }" https://connection.keboola.com/v2/storage/buckets/in.c-main/tables-async -{% endhighlight %} - -This will create an asynchronous job, importing data from the `192726698` file into the `new-table` destination table in the `in.c-main` bucket. -Then [poll for the job results](/integrate/jobs/#job-polling), or review its status in the UI. - -#### Python Example -The above process is implemented in the following example script in Python. This script uses the -[Requests](https://2.python-requests.org/en/master/) library for sending HTTP requests and -the [Boto 3](https://github.com/boto/boto3) library for working with Amazon S3. Both libraries can be -installed using pip: - -{% highlight bash %} -pip install boto3 -pip install requests -{% endhighlight %} - -{% highlight python %} -{% include async-create.py %} -{% endhighlight %} - -#### Upload Files Using Storage API Importer -For production setup, we recommend using the approach [outlined above](#manually-uploading-a-file) -with direct upload to S3 as it is more reliable and universal. -In case you need to avoid using an S3 client, it is also possible to upload the -file by a simple HTTP request to [Storage API Importer Service](/integrate/storage/api/importer/). - -{% highlight bash %} -curl --request POST --header "X-StorageApi-Token:storage-token" --form "data=@new-file.csv" https://import.keboola.com/upload-file -{% endhighlight %} - -The above will return a response similar to this: - -{% highlight json %} -{ - "id": 418137780, - "created": "2018-07-17T13:48:57+0200", - "isPublic": false, - "isSliced": false, - "isEncrypted": true, - "name": "404.md", - "url": "https:\/\/kbc-sapi-files.s3.amazonaws.com\/exp-15\/4088\/files\/2018\/07\/17\/418137779.new-file.csv...truncated", - "region": "us-east-1", - "sizeBytes": 1765, - "tags": [], - "maxAgeDays": 15, - "runId": null, - "runIds": [], - "creatorToken": { - "id": 144880, - "description": "file upload" - } -} -{% endhighlight %} - -After that, import the file into Table Storage by calling either -[Create Table API call](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/buckets/-id-/tables-async) -(for a new table) or -[Load Data API call](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/tables/-id-/import-async) -(for an existing table). - -### Working with Sliced Files -Depending on the backend and table size, the data file may be sliced into chunks. -Requirements for uploading sliced files are described in the respective part of the -[API documentation](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/files/prepare). - -When you attempt to download a sliced file, you will instead obtain its manifest -listing the individual parts. Download the parts individually and join them -together. For a reference implementation of this process, see -our [TableExporter class](https://github.com/keboola/storage-api-php-client/blob/master/src/Keboola/StorageApi/TableExporter.php). - -**Important:** When exporting a table through the *Table* --- *Export* UI, the file will -be already merged and listed in the *File Uploads* section with the `storage-merged-export` tag. - -If you want to download a sliced file, [get credentials](https://api.keboola.com/?service=storage#get-/v2/storage/branch/-branchId-/files/-fileId-) -to download the file from AWS S3. Assuming that the file ID is 192611596, for example, call - -{% highlight bash %} -curl --header "X-StorageAPI-Token: storage-token" https://connection.keboola.com/v2/storage/files/192611596?federationToken=1 -{% endhighlight %} - -which will return a response similar to this: - -{% highlight json %} -{ - "id": 192611596, - "created": "2016-06-21T15:25:35+0200", - "name": "in.c-redshift.blog-data.csv", - "url": "https://s3.amazonaws.com/kbc-sapi-files/exp-2/578/table-exports/in/c-redshift/blog-data/192611594.csvmanifest?X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAJ2N244XSWYVVYVLQ%2F20160621%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20160621T135137Z&X-Amz-SignedHeaders=host&X-Amz-Expires=3600&X-Amz-Signature=ee69d94f0af06bcf924df0f710dcd92e6503a13c8a11a86be2606552bf9a8b26", - "region": "us-east-1", - "sizeBytes": 24541, - "tags": [ - "table-export" - ], - ... - "s3Path": { - "bucket": "kbc-sapi-files", - "key": "exp-2/578/table-exports/in/c-redshift/blog-data/192611594.csv" - }, - "credentials": { - "AccessKeyId": "ASI...UQQ", - "SecretAccessKey": "LHU...HAp", - "SessionToken": "Ago...uwU=", - "Expiration": "2016-06-22T01:51:37+00:00" - } -} -{% endhighlight %} - -The field `url` contains the URL to the file manifest. Upon downloading it, you will get a JSON file with contents -similar to this: - -{% highlight json %} -{ - "entries": [ - {"url":"s3://kbc-sapi-files/exp-2/578/table-exports/in/c-redshift/blog-data/192611594.csv0000_part_00"}, - {"url":"s3://kbc-sapi-files/exp-2/578/table-exports/in/c-redshift/blog-data/192611594.csv0001_part_00"} - ] -} -{% endhighlight %} - -Now you can download the actual data file slices. URLs are provided in the manifest file, and credentials to them -are returned as part of the previous file info call. To download the files from S3, you need an S3 client. There -are a wide number of clients available; for example, use the -[S3 AWS command line client](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html). Before -using it, [pass the credentials](https://docs.aws.amazon.com/cli/latest/topic/config-vars.html#credentials) -by executing , for instance, the following commands - -on *nix systems: -{% highlight bash %} -export AWS_ACCESS_KEY_ID=ASI...UQQ -export AWS_SECRET_ACCESS_KEY=LHU...HAp -export AWS_SESSION_TOKEN=Ago...wU= -{% endhighlight %} - -or on Windows: -{% highlight bash %} -SET AWS_ACCESS_KEY_ID=ASI...UQQ -SET AWS_SECRET_ACCESS_KEY=LHU...HAp -SET AWS_SESSION_TOKEN=Ago...wU= -{% endhighlight %} - -Then you can actually download the files by executing the AWS S3 CLI [cp command](https://docs.aws.amazon.com/cli/latest/reference/s3/cp.html): -{% highlight bash %} -aws s3 cp s3://kbc-sapi-files/exp-2/578/table-exports/in/c-redshift/blog-data/192611594.csv0000_part_00 192611594.csv0000_part_00 -aws s3 cp s3://kbc-sapi-files/exp-2/578/table-exports/in/c-redshift/blog-data/192611594.csv0001_part_00 192611594.csv0001_part_00 -{% endhighlight %} - -After that, merge the files together by executing the following commands - -on *nix systems: -{% highlight bash %} -cat 192611594.csv0000_part_00 192611594.csv0001_part_00 > merged.csv -{% endhighlight %} - -or on Windows: -{% highlight bash %} -copy 192611594.csv0000_part_00 /B +192611594.csv0001_part_00 /B merged2.csv -{% endhighlight %} +--- +title: Manually Importing and Exporting Data +permalink: /integrate/storage/api/import-export/ +redirect_to: https://help.keboola.com/storage/api/import-export/ +--- diff --git a/integrate/storage/api/importer.md b/integrate/storage/api/importer.md index 1797a1de..56e30bf8 100644 --- a/integrate/storage/api/importer.md +++ b/integrate/storage/api/importer.md @@ -1,50 +1,5 @@ ---- -title: Storage API Importer -permalink: /integrate/storage/api/importer/ ---- - -* TOC -{:toc} - -The [whole process of importing](/integrate/storage/api/) a table into Storage can be simplified with the -Storage API Importer Service. -The Storage API Importer allows you to make an HTTP POST request and import a file directly into an existing Storage table. - -The HTTP request must contain the `tableId` and `data` form fields. The specified table must already exist in [Storage](https://help.keboola.com/storage/). -Therefore to upload the `my-table.csv` CSV file (and replace the contents) into the `my-table` table in the `in.c-main` bucket, -call: - -{% highlight bash %} -curl --request POST --header "X-StorageApi-Token:storage-token" --form "tableId=in.c-main.my-table" --form "data=@my-table.csv" "https://import.keboola.com/write-table" -{% endhighlight %} - -Using the Storage API Importer is the easiest way to upload data into Storage (except for -using one of the [API clients](/integrate/storage/#clients)). However, the disadvantage is that the whole data file -has to be posted in a single HTTP request. **The maximum limit for a file size is 2GB and the transfer time is 45 minutes**. -This means that for substantially large files (usually more than hundreds of MB) -you may experience timeouts. If that happens, use the above outlined approach and upload the -files [directly to S3](/integrate/storage/api/import-export/#manually-uploading-a-file). - -## Parameters - -- `tableId` (required) Storage Table ID, example: in.c-main.users -- `data` (required) Uploaded CSV file. Raw file or compressed by [gzip](http://www.gzip.org/) -- `delimiter` (optional) Field delimiter used in a CSV file. The default value is ' , '. Use '\t' or type the tab char for tabulator. -- `enclosure` (optional) Field enclosure used in a CSV file. The default value is '"'. -- `escapedBy` (optional) CSV escape character; empty by default. -- `incremental` (optional) If incremental is set to 0 (its default), the target table is truncated before each import. - -Full list of avaialable parameters is available in the [API documentation](https://api.keboola.com/?service=import#import). - -## Examples -To load data incrementally (append new data to existing contents): - -{% highlight bash %} -curl --request POST --header "X-StorageApi-Token:storage-token" --form "incremental=1" --form "tableId=in.c-main.my-table" --form "data=@my-table.csv" "https://import.keboola.com/write-table" -{% endhighlight %} - -To load data with a non-default delimiter (tabulator) and enclosure (empty): - -{% highlight bash %} -curl --request POST --header "X-StorageApi-Token:storage-token" --form "delimiter=\t" --form "enclosure=" --form "tableId=in.c-main.my-table" --form "data=@my-table.csv" "https://import.keboola.com/write-table" -{% endhighlight %} +--- +title: Storage API Importer +permalink: /integrate/storage/api/importer/ +redirect_to: https://help.keboola.com/storage/api/importer/ +--- diff --git a/integrate/storage/api/index.md b/integrate/storage/api/index.md index f96dbb5c..2ed85269 100644 --- a/integrate/storage/api/index.md +++ b/integrate/storage/api/index.md @@ -1,36 +1,5 @@ ---- -title: Storage API -permalink: /integrate/storage/api/ ---- - -* TOC -{:toc} - -If you are new to Keboola, you should make yourself familiar with -the [Storage component](https://help.keboola.com/storage/) before you start using it. -For a general introduction to working with Keboola APIs, see the [API Introduction](/overview/api/). -[Storage API](https://api.keboola.com/?service=storage) provides a number of functions. These are the most important ones: - -- [Component configurations](https://api.keboola.com/?service=storage#tag--Component-Configurations) -- [Storage tables](https://api.keboola.com/?service=storage#tag--Tables) -- [File uploads](https://api.keboola.com/?service=storage#tag--Files) -- [Storage buckets](https://api.keboola.com/?service=storage#tag--Buckets) - -Virtually, all API calls require a [Storage API token](https://help.keboola.com/storage/tokens/) to -be passed as the `X-StorageApi-Token` header. -Please note that the Storage API calls require the request to be sent -as `form-data` (unlike the rest of Keboola API, which is sent as `application/json`). - -For exporting tables from and importing tables to Storage, we highly recommend that you use one of the -[available clients](/integrate/storage/) or the [Storage API Importer service](/integrate/storage/api/importer/). -All imports and exports are done using CSV files. See -the [RFC4180 Specification](https://tools.ietf.org/html/rfc4180) for the format -and encoding specification, and -[User documentation](https://help.keboola.com/storage/tables/csv-files/) for help on how to create such files. - -Continue reading the following sections for guidance on how to get started: - -- [Storage importer service for the easiest upload of data via API](/integrate/storage/api/importer/) -- [Getting started with component configurations](/integrate/storage/api/configurations/) -- [Importing and exporting data](/integrate/storage/api/import-export/) -- [TDE exporter for exporting data to Tableau Data Extracts](/integrate/storage/api/tde-exporter/) +--- +title: Storage API +permalink: /integrate/storage/api/ +redirect_to: https://help.keboola.com/storage/api/ +--- diff --git a/integrate/storage/api/tde-exporter.md b/integrate/storage/api/tde-exporter.md index be82c40d..b60e0aab 100644 --- a/integrate/storage/api/tde-exporter.md +++ b/integrate/storage/api/tde-exporter.md @@ -1,95 +1,5 @@ ---- -title: TDE Exporter -permalink: /integrate/storage/api/tde-exporter/ ---- - -* TOC -{:toc} - -[TDE Exporter](https://github.com/keboola/tde-exporter) exports tables from Keboola Storage into the -[TDE file format (Tableau Data Extract)](https://www.tableau.com/about/blog/2014/7/understanding-tableau-data-extracts-part1). -This component is normally a part of the [Tableau Writer](https://help.keboola.com/tutorial/write/), -but it can also be used as a standalone component. - -Users can [run a TDE exporter job](/integrate/jobs/) as any other Keboola component or register it -as an orchestration task. After the exporter finishes, the resulting TDE files will be available in the -*Storage* --- *File uploads* section where you can download them via UI or [API](/integrate/storage/api/import-export/). - -## Running the Component -The TDE Exporter is a Keboola [component](https://help.keboola.com/extend/component/) supporting both -[stored](/integrate/storage/api/configurations/) and -custom configurations supplied directly in the `run` request. - -### Stored Configuration -To run the TDE exporter with a stored configuration, first -[create the configuration](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/components/-componentId-/configs). -See [below](#custom-configuration) for the required configuration contents. -This call will give you the ID of the newly created configuration (for instance, `new-configuration-id`). -Then [create a job](/integrate/jobs/) with the specified configuration: - -{% highlight json %} -{ - "config": "new-configuration-id" -} -{% endhighlight %} - -### Custom Configuration -You can specify the entire configuration in the API call. The JSON configuration conforms -to the [general configuration format](https://help.keboola.com/extend/common-interface/config-file/). The specific part -is only the `parameters` section. A sample request to the `in.c-main.old-table` export table would look like this: - -{% highlight json %} -{ - "configData": { - "storage": { - "input": { - "tables": [{ - "source": "in.c-main.old-table" - }] - } - }, - "parameters": { - "tags": ["sometag"], - "typedefs": { - "in.c-main.old-table": { - "id": { - "type": "number" - }, - "col1": { - "type": "string" - } - } - } - } - } -} -{% endhighlight %} - -The `parameters` section contains: - -- `tags`: array of tags that will be assigned to the resulting file in Storage File Uploads. -- `typedefs`: definitions of data types mapping source tables columns to destination TDE columns. - -The type definitions are entered as an object whose name must match the name of the table in the -`storage.input.tables.source` node (`in.c-main.old-table` in the above example). Object properties -are names of the table columns; each must have the `type` property which is one of the -[supported column types](https://help.tableau.com/current/pro/desktop/en-us/datafields_typesandroles_datatypes.htm): -`boolean`, `number`, `decimal`, `date`, `datetime` and `string`. - -## Date and DateTime -Data for these data types can be specified in the format used -in the [strptime function](https://pubs.opengroup.org/onlinepubs/009695399/functions/strptime.html). The format is specified as part of the column's type definition. For example: - -{% highlight json %} -{ - "col1": { - "type": "date", - "format":"%m-%d-%Y" - } -} -{% endhighlight %} - -If no format is specified, the following default formats are used: - -- For `date`: `%Y-%m-%d` -- For `datetime`: `%Y-%m-%d %H:%M:%S or %Y-%m-%d %H:%M:%S.%f` +--- +title: TDE Exporter +permalink: /integrate/storage/api/tde-exporter/ +redirect_to: https://help.keboola.com/storage/api/tde-exporter/ +--- diff --git a/integrate/storage/docker-cli-client.md b/integrate/storage/docker-cli-client.md index a5897cff..5f0e07b0 100644 --- a/integrate/storage/docker-cli-client.md +++ b/integrate/storage/docker-cli-client.md @@ -2,112 +2,5 @@ title: Storage Docker CLI Client permalink: /integrate/storage/docker-cli-client/ redirect_from: /integrate/storage/php-cli-client/ +redirect_to: https://help.keboola.com/storage/api/clients/docker-cli/ --- - -* TOC -{:toc} - -The Storage API Docker command line interface (CLI) client is a portable command line client which provides -a simple implementation of [Storage API](https://api.keboola.com/?service=storage). -It runs on any platform which has Docker installed. - -Currently, the client implements - -- functions for exporting and importing tables; -- functions for creating and deleting buckets; and additionally, -- the [project backup feature](https://help.keboola.com/management/project-export/). - -The client source is available in our [Github repository](https://github.com/keboola/storage-api-cli). -The client docker image is available in the [Quay repository](https://quay.io/repository/keboola/storage-api-cli?tab=tags). - -## Running in Docker -To print available commands: - -{% highlight bash %} -docker run quay.io/keboola/storage-api-cli:latest -{% endhighlight %} - -The `latest` image tag always refers to the latest tagged version. - -## Running Phar - -PHAR (PHP Archive) is now deprecated, but there are still some older versions available. See the [repository documentation](https://github.com/keboola/storage-api-cli#running-phar-deprecated). - -### Example --- Creating a Table -To create a new table in Storage, use the `create-table` command. Provide the name of an -existing bucket, the name of the new table and a CSV file with the table's contents. - -To create the`new-table` table in the `in.c-main` bucket, use - -{% highlight bash %} -docker run --volume=$("pwd"):/data quay.io/keboola/storage-api-cli:latest create-table in.c-main new-table /data/new-table.csv --token=storage_token -{% endhighlight %} - -or on Windows: - - docker run --volume=C:\Users\name\some-dir:/data quay.io/keboola/storage-api-cli:latest create-table in.c-main new-table /data/new-table.csv --token=storage_token - -or when using other then [default US region](/overview/api/#regions-and-endpoints), you need to provide the Storage API address: - -{% highlight bash %} -docker run --volume=$("pwd"):/data quay.io/keboola/storage-api-cli:latest create-table in.c-main new-table /data/new-table.csv --token=storage_token --url="https://connection.eu-central-1.keboola.com/" -{% endhighlight %} - -Any of the above commands will import the contents of `new-table.csv` in the current directory into the newly -created table. You should see an output similar to this one: - - Authorized as: ondrej.popelka@keboola.com (Odinuv Sandbox) - Bucket found ok - Table create start - Table create end - Table id: in.c-main.new-table - -*Please note that the Docker container can only access folders within the container, so you need to mount a local folder. -In the example above, the local folder `$("pwd")` (replaced by the absolute path at runtime) is mounted as `/data` into the container. -The table is then accessible in this folder. The same approach applies to all other commands working with local files.* - -### Example --- Importing Data -If you only want to import new data into the table, use the `write-table` command and provide -the ID (*bucketName.tableName*) of an existing table. - -To import data into the `new-table` table in the `in.c-main` bucket, use - -{% highlight bash %} -docker run --volume=$("pwd"):/data quay.io/keboola/storage-api-cli:latest write-table in.c-main.new-table /data/new-data.csv --token=storage_token --incremental -{% endhighlight %} - -The above command will import the contents of the `new-data.csv` file into the existing table. If the -`--incremental` parameter is supplied, the table contents will be appended. If the parameter is not -supplied, the table contents will be overwritten. You should see an output similar to this one: - - Authorized as: ondrej.popelka@keboola.com (Tutorial) - Table found ok - Import start - Import done in 17 secs. - - Results: - transaction: - warnings: - importedColumns: - - id - - secondCol - totalRowsCount: 8 - totalDataSizeBytes: 4096 - - -### Example --- Exporting Data -If you want to export a table from Storage, use the `export-table` command. Provide -the ID (*bucketName.tableName*) of an existing table. - -To export data from the `old-table` table in the `in.c-main` bucket, use - -{% highlight bash %} -docker run --volume=$("pwd"):/data quay.io/keboola/storage-api-cli:latest export-table in.c-main.old-table /data/old-data.csv --token=storage_token -{% endhighlight %} - -The above command will export the table from Storage and save it as `old-data.csv` in -the current directory. You should see an output similar to this one: - - Authorized as: ondrej.popelka@keboola.com (Tutorial) - Table found ok - Export done in 17 secs. diff --git a/integrate/storage/index.md b/integrate/storage/index.md index 4812d9fb..6ddd8beb 100644 --- a/integrate/storage/index.md +++ b/integrate/storage/index.md @@ -1,58 +1,5 @@ --- title: Storage permalink: /integrate/storage/ +redirect_to: https://help.keboola.com/storage/api/ --- - -* TOC -{:toc} - -As the central Keboola component, Storage - -- Keeps all data in [**buckets** and **tables**](https://help.keboola.com/storage/); -- Controls access to the data using **tokens**; -- Logs all data manipulations as **events**; -- Maintains the index of all other Keboola **components** and stores their **configurations**. - -All this (and a few other things) is available through [Storage API (SAPI)](https://api.keboola.com/?service=storage). -To authorize access to a specific project, most calls to Storage API require -a [Storage API Token](https://help.keboola.com/storage/tokens/) along with your request. -It is required regardless of whether you use the bare API or any of the clients. - -## Storage API Clients -Although you can work directly with the API, we recommend using one of our Storage API clients, as they simplify some tasks. -There are four Storage clients with different feature sets available: - -1. [PHP client library](https://github.com/keboola/storage-api-php-client) --- a PHP library supporting most of the Storage API features; -use it programmatically in PHP. -2. [R client library](/integrate/storage/r-client/) --- an R library supporting most data manipulation features of the Storage API; -use it programmatically in R. -3. [Python client library](/integrate/storage/python-client/) --- a Python library supporting most data manipulation features and -workspace manipulation features of the Storage API; use it programmatically in Python. -4. [Docker CLI client](https://github.com/keboola/storage-api-cli) --- a CLI (command line interface) application supporting -basic data manipulation features of the Storage API; use it from the command line provided that you have Docker available. - -Additional tools: - -- [Storage API Console](https://storage-api-console.keboola.com/) --- a UI to work with Keboola Storage; -this is accessible to anyone with a Storage Token (not necessarily a Keboola project administrator) -- [Table Importer Service](/integrate/storage/api/importer/) --- a service designed for simplified table loads - -The client choice is purely up to you, but it is best to use the most straightforward solution. - -## Table Imports and Exports -Tables are imported to and exported from Storage via asynchronous (background) jobs. - -- When importing a table, the actual data is first transported to an Amazon S3 storage, -and then bulk is loaded into the internal database in Storage. Similarly, -- When exporting a table, the data is first offloaded to an Amazon S3 storage and downloaded from there. - -While this process is much more complicated than a simple file upload or download, -it offers better **manageability** and **traceability** features. -Use one of the above mentioned clients to import and export your data. The client will handle the entire process -without you worrying about the technical details. - -If all you need is to import data into Storage (for example, for project prototyping), you may -also use the [Storage Importer Service](/integrate/storage/api/importer/). - -Still interested in handling the file uploads/downloads manually? -[Read on](/integrate/storage/api/import-export/). diff --git a/integrate/storage/new-table.csv b/integrate/storage/new-table.csv deleted file mode 100644 index 8dbb6c46..00000000 --- a/integrate/storage/new-table.csv +++ /dev/null @@ -1,5 +0,0 @@ -"id","secondCol" -"1","a" -"2","b" -"3","c" -"4","d" \ No newline at end of file diff --git a/integrate/storage/php-client.md b/integrate/storage/php-client.md index 257a8164..882d7c63 100644 --- a/integrate/storage/php-client.md +++ b/integrate/storage/php-client.md @@ -1,149 +1,5 @@ ---- -title: Storage PHP Client Library -permalink: /integrate/storage/php-client/ ---- - -* TOC -{:toc} - -The Storage API PHP client library is a portable command line client providing -the most complete [Storage API](https://api.keboola.com/?service=storage) implementation. -It runs on any platform which has PHP installed. -Currently this client implements almost all Storage API functions including, of course, exporting and importing tables. - -The client source is available in our [Github repository](https://github.com/keboola/storage-api-php-client). - -## Installation - -The Library is available as a [Composer package](https://getcomposer.org/). -Unless you already have it, [install Composer](https://getcomposer.org/download/) on your system. -On *nix system, do so by running - -{% highlight bash %} -curl -s http://getcomposer.org/installer | php -mv ./composer.phar ~/bin/composer # or /usr/local/bin/composer -{% endhighlight %} - -On Windows, use the [installer](https://getcomposer.org/Composer-Setup.exe). - -To install the library, run - -{% highlight bash %} -composer require keboola/storage-api-client -{% endhighlight %} - -in the root of your project. You should get an output similar to this one: - - Using version ^4.11 for keboola/storage-api-client - ./composer.json has been created - Loading composer repositories with package information - Updating dependencies (including require-dev) - - Installing aws/aws-sdk-php (3.18.18) - Downloading: 100% - ... - - Installing keboola/storage-api-client (4.11.0) - Downloading: 100% - Writing lock file - Generating autoload files - -Then add the generated autoloader in your bootstrap script: - -{% highlight php %} -require 'vendor/autoload.php'; -{% endhighlight %} - -You can read more in the [Composer documentation](https://getcomposer.org/doc/01-basic-usage.md). Packages -installable by Composer can be browsed at [Packagist package repository](https://packagist.org/). - -## Usage -The Storage API client is implemented as a single class. To create an instance of the class, provide a Storage API token to the -constructor. - -{% highlight php %} - 'your-token', -]); -{% endhighlight %} - - -### Example --- Create a Table -To create a new table in Storage, it is recommended to use an additional -[php-csv](https://github.com/keboola/php-csv) library to work -with CSV files. The library will get installed -automatically with the Storage API client, so you can use it out of the box. -To create a new table and import CSV data in it, use the following PHP script: - -{% highlight php %} - 'your-token', -]); -$csvFile = new CsvFile('./new-table.csv'); -$client->createTableAsync('in.c-main', 'new-table', $csvFile); -{% endhighlight %} - -### Example --- Import Data -To import CSV data into an existing table and overwrite its contents, use the following PHP script: - -{% highlight php %} - 'your-token', -]); -$csvFile = new CsvFile('./new-table.csv'); -$client->writeTableAsync('in.c-main.new-table', $csvFile); -{% endhighlight %} - -### Example --- Import Data Incrementally -To import CSV data into an existing table and append the new data to the existing table contents, use the following PHP script: - -{% highlight php %} - 'your-token', -]); -$csvFile = new CsvFile('./new-table.csv'); -$client->writeTableAsync('in.c-main.new-table', $csvFile, ['incremental' => true]); -{% endhighlight %} - -All available upload options are listed in the [API documentation](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/tables/-id-/import-async). - -### Example --- Export Data -To export data from a Storage table to a CSV file, use the -`TableExporter` class. It is part of the client library. You can use the following script: - -{% highlight php %} - 'your-token' -]); - -$exporter = new TableExporter($client); -$exporter->exportTable('in.c-main.my-table', './old-table.csv'); -{% endhighlight %} +--- +title: Storage PHP Client Library +permalink: /integrate/storage/php-client/ +redirect_to: https://help.keboola.com/storage/api/clients/php-client/ +--- diff --git a/integrate/storage/python-client.md b/integrate/storage/python-client.md index 12df05a1..27bc5379 100644 --- a/integrate/storage/python-client.md +++ b/integrate/storage/python-client.md @@ -1,118 +1,5 @@ --- title: Python Client Library permalink: /integrate/storage/python-client/ +redirect_to: https://help.keboola.com/storage/api/clients/python-client/ --- - -* TOC -{:toc} - -The Python client library is a [Storage API client](https://api.keboola.com/?service=storage) which you can use in your Python code. -The current implementation supports all basic data manipulations: - -- Importing data -- Exporting data -- Creating and deleting buckets and tables -- Creating and deleting workspaces - -The client source code is available in our [Github repository](https://github.com/keboola/sapi-python-client/). - -## Installation -This library is available on [Github](https://github.com/keboola/sapi-python-client), so we -recommend that you use the `pip` package to install it: - - pip3 install git+https://github.com/keboola/sapi-python-client.git - -## Usage -The client contains a `Client` class, which encapsulates all API endpoints and holds a storage token and URL. Each API endpoint is -represented by its own class (`Files`, `Buckets`, `Jobs`, etc.), which can be used standalone if you only work with one endpoint. -This means that the two following examples are equivalent: - -{% highlight python %} -from kbcstorage.client import Client - -client = Client('https://connection.keboola.com', 'your-token') -client.tables.detail('in.c-demo.some-table') -{% endhighlight %} - -{% highlight python %} -from kbcstorage.tables import Tables - -tables = Tables('https://connection.keboola.com', 'your-token') -tables.detail('in.c-demo.some-table') -{% endhighlight %} - -### Example --- Create Table and Import Data -To create a new table in Storage, use the `create` function of the `Tables` class. Provide the name of an existing bucket, -the name of the new table and a CSV file with the table's contents. - -To create the `new-table` table in the `in.c-main` bucket, use: - -{% highlight python %} -from kbcstorage.client import Client - -client = Client('https://connection.keboola.com', 'your-token') -client.tables.create(name='new-table', - bucket_id='in.c-main', - file_path='coords.csv', - primary_key=['id']) -{% endhighlight %} - -The above command will import the contents of the `coords.csv` file into the newly created table. It will -also mark the `id` column as the primary key. -### Example --- Load to existing table, incrementally - -To load data incrementally into an existing table, we can use the [load](https://github.com/keboola/sapi-python-client/blob/5a93926c2191ccd6b7402c9e24d9912884d87d4c/kbcstorage/tables.py#L207) method, where `table_id` is the ID of the table that you want to load into, and `path` is the path to your csv file containing the data: - -{% highlight python %} - -from kbcstorage.client import Client - -client = Client('https://connection.keboola.com', 'your-token') - -client.tables.load(table_id=table_id, file_path=path, is_incremental=True) - -{% endhighlight %} -### Example --- Export Data -To export data from the `old-table` table in the `in.c-main` bucket, use: - -{% highlight python %} -from kbcstorage.client import Client -import csv - -client = Client('https://connection.keboola.com', 'your-token') -client.tables.export_to_file(table_id='in.c-main.new-table', path_name='.') -with open('./new-table', mode='rt', encoding='utf-8') as in_file: - lazy_lines = (line.replace('\0', '') for line in in_file) - reader = csv.reader(lazy_lines, lineterminator='\n') - for row in reader: - print(row) -{% endhighlight %} - -The above command will export the table from Storage into the file `new-table` and read it using -[CSV Reader](https://docs.python.org/3.6/library/csv.html#reader-objects). - -### Other Examples - -{% highlight python %} -# create a client -client = Client('https://connection.keboola.com', 'your-token') - -# create a bucket -client.buckets.create(name='demo', stage='in') - -# list buckets -client.buckets.list() - -# list all tables -client.tables.list() - -# list all tables in a bucket -client.buckets.list_tables(bucket_id='in.c-demo') - -# delete a table -client.tables.delete(table_id='in.c-demo.some-table') - -# delete a bucket -client.buckets.delete(bucket_id='in.c-main', force=True) - -{% endhighlight %} diff --git a/integrate/storage/r-client.md b/integrate/storage/r-client.md index d8b46737..56f682f2 100644 --- a/integrate/storage/r-client.md +++ b/integrate/storage/r-client.md @@ -1,123 +1,5 @@ ---- -title: R Client Library -permalink: /integrate/storage/r-client/ ---- - -* TOC -{:toc} - -The R client library is a [Storage API client](https://api.keboola.com/?service=storage) which you can use in your R code. -The current implementation supports all basic data manipulations: - -- Importing data -- Exporting data -- Creating and deleting buckets and tables - -The client source code is available in our [Github repository](https://github.com/keboola/sapi-r-client). - -## Installation -This library is available on [Github](https://github.com/keboola/sapi-r-client), so we -recommend that you use the `devtools` package to install it. - -{% highlight r %} -# first install the devtools package if it isn't already installed -install.packages("devtools") - -# install dependencies (another github package for aws requests) -devtools::install_github("cloudyr/aws.s3") - -# install the SAPI R client package -devtools::install_github("keboola/sapi-r-client") - -# load the library (dependencies will be loaded automatically) -library(keboola.sapi.r.client) -{% endhighlight %} - -## Usage -To list available commands, run -{% highlight r %} -?keboola.sapi.r.client::SapiClient -{% endhighlight %} - -**Important**: If you are running the code in R Studio, it might require a restart so that its help index is updated -and the above command works. - -The client is implemented as an [RC class](http://adv-r.had.co.nz/R5.html). To work with it, create an instance of the client. -The only required argument to create it is a valid Storage API token. - -{% highlight r %} -client <- SapiClient$new( - token = 'your-token' -) -{% endhighlight %} - -### Example --- Create a Table and Import Data -To create a new table in Storage, use the `saveTable` function. Provide the name of an existing bucket, -the name of the new table and a CSV file with the table's contents. - -To create the `new-table` table in the `in.c-main` bucket, use - -{% highlight r %} -myDataFrame <- data.frame(id = c(1,2,3,4), secondCol = c('a', 'b', 'c', 'd')) -client <- SapiClient$new( - token = 'your-token' -) - -table <- client$saveTable( - df = myDataFrame, - bucket = "in.c-main", - tableName = "new-table", - options = list(primaryKey = 'id') -) -{% endhighlight %} - -The above command will import the contents of the `myDataFrame` variable into the newly created table. It will -also mark the `id` column as the primary key. - -### Example --- Export Data -If you want to export a table from Storage and import it into R, use the `importTable` function. Provide -the ID (*bucketName.tableName*) of an existing table. - -To export data from the `old-table` table in the `in.c-main` bucket, use - -{% highlight r %} -client <- SapiClient$new( - token = 'your-token' -) - -data <- client$importTable('in.c-main.old-table') -{% endhighlight %} - -The above command will export the table from Storage and save it in the `data` variable. The output is -a [data.table](https://cran.r-project.org/web/packages/data.table/index.html) object compatible with a `data.frame`. - -### Other Examples - -{% highlight r %} -# create a client -client <- SapiClient$new( - token = 'your-token' -) - -# verify the token -tokenDetails <- client$verifyToken() - -# create a bucket -bucket <- client$createBucket("new_bucket", "in", "A brand new Bucket!") - -# list buckets -buckets <- client$listBuckets() - -# list all tables -tables <- client$listTables() - -# list all tables in a bucket -tables <- client$listTables(bucket = bucket$id) - -# delete a table -client$deleteTable(table$id) - -# delete a bucket -client$deleteBucket(bucket$id) - -{% endhighlight %} +--- +title: R Client Library +permalink: /integrate/storage/r-client/ +redirect_to: https://help.keboola.com/storage/api/clients/r-client/ +--- diff --git a/integrate/storage/sys.c-table-importer.test-config.csv b/integrate/storage/sys.c-table-importer.test-config.csv deleted file mode 100644 index c27017d8..00000000 --- a/integrate/storage/sys.c-table-importer.test-config.csv +++ /dev/null @@ -1,2 +0,0 @@ -"table","primaryKey","incremental","enclosure","delimiter","escapedBy","tag","rowId" -"in.c-main.new-table","","0","""",",","","new-data","1" diff --git a/overview/api/index.md b/overview/api/index.md index 107dc3bf..eb65e91b 100644 --- a/overview/api/index.md +++ b/overview/api/index.md @@ -23,14 +23,14 @@ Using a portal for a different stack than your token's stack will result in