Databricks End Points

Edit online

Summary

These end point are used to manage Databricks integration within Cloudability that support the creation, updating and deletion of Databricks credentials within Cloudability.

The Credential Object

  • orgId (string) : Organisation Id
  • accountId (string) : Databricks Account Id
  • secret (string) - Secret created for the Service Principal User
  • clientId (string) - Client Id created for the Service Principal User
  • endpoint (string) - Databricks account endpoint
  • rateLimit (int) - This rate limit is fixed for the type of endpoints in Databricks

End Points

POST v3/vendors/databricks/accounts

This API is used for creating a new Databricks credential.

Example Request 

{
"vendorAccountId": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"type": "databricks_user",
"client_id": "b40563d6-d7de-4128-a542-c2f50c2a7ab8",
"workspace": "apptio-yipit-pipeline.cloud.databricks.com",
"warehouse_id": "f020534a981405ab",
"subscriptionModel": "Marketplace",
"dependentCsp": "Azure",
"activePaymentModel": "Pay as you go",
"contractDetails": [
{
"endDate": "2025-06-15",
"paymentModel": "Committed Contract",
"discount": 20,
"startDate": "2024-01-01"
},
{
"endDate": "2025-07-15",
"paymentModel": "Committed Contract",
"discount": 20,
"startDate": "2024-02-01"
}
],
"secret": "dose029cb6a9dedca9b2ac14955e442260d2"
}

Example Response 

{
"result": {
"id": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"vendorAccountName": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"vendorAccountId": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"vendorKey": "databricks",
"verification": {
"state": "unverified"
},
"authorization": {
"type": "databricks_user",
"client_id": "b40563d6-d7de-4128-a542-c2f50c2a7ab8", "workspace": "apptio-yipit-pipeline.cloud.databricks.com", "warehouse_id": "f020534a981405ab",
"subscriptionModel": "Marketplace",
"dependentCsp": "Azure",
"activePaymentModel": "Pay as you go",
"contractDetails": [
{
"startDate": "2024-01-01",
"endDate": "2025-06-15",
"discount": 20,
"paymentModel": "Committed Contract"
},
{
"startDate": "2024-02-01",
"endDate": "2025-07-15",
"discount": 20,
"paymentModel": "Committed Contract"
}
]
},
"createdAt": "2024-03-20T13:55:57Z",
"consumerOrgId": ""
}
}

PUT v3/vendors/databricks/accounts/:accountId

This API is used for updating the credential with the given ID

Example Request 

{
"vendorAccountId": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"type": "databricks_user",
"client_id": "b40563d6-d7de-4128-a542-c2f50c2a7ab8",
"workspace": "apptio-yipit-pipeline.cloud.databricks.com",
"warehouse_id": "f020534a981405ab",
"subscriptionModel": "Marketplace",
"dependentCsp": "Azure",
"activePaymentModel": "Pay as you go",
"contractDetails": [
{
"endDate": "2025-06-15",
"paymentModel": "Committed Contract",
"discount": 50,
"startDate": "2024-01-01"
},
{
"endDate": "2025-07-15",
"paymentModel": "Committed Contract",
"discount": 50,
"startDate": "2024-02-01"
}
],
"secret": "dose029cb6a9dedca9b2ac14955e442260d2"
}

Example Response 

{
"result": {
"id": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"vendorAccountName": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"vendorAccountId": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"vendorKey": "databricks",
"verification": {
"state": "unverified"
},
"authorization": {
"type": "databricks_user",
"client_id": "b40563d6-d7de-4128-a542-c2f50c2a7ab8", "workspace": "apptio-yipit-pipeline.cloud.databricks.com", "warehouse_id": "f020534a981405ab","subscriptionModel": "Marketplace",
"dependentCsp": "AWS",
"activePaymentModel": "Pay as you go",
"contractDetails": [
{
"startDate": "2024-01-01",
"endDate": "2025-06-15",
"discount": 50,
"paymentModel": "Committed Contract"
},
{
"startDate": "2024-02-01",
"endDate": "2025-07-15",
"discount": 50,
"paymentModel": "Committed Contract"
}
]
},
"createdAt": "2024-03-20T13:55:57Z",
"consumerOrgId": ""
}
}

GET v3/vendors/databricks/accounts/:accountId

This API is used for returning Databricks credentials for the given account.

Example Response 

{
"result": [
{
"id": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"vendorAccountName": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"vendorAccountId": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"vendorKey": "databricks",
"verification": {
"state": "verified",
"lastVerificationAttemptedAt": "2024-03-20T13:10:28Z"
},
"authorization": {
"type": "databricks_user",
"client_id": "b40563d6-d7de-4128-a542-c2f50c2a7ab8", "workspace": "apptio-yipit-pipeline.cloud.databricks.com", "warehouse_id": "f020534a981405ab",
"subscriptionModel": "Marketplace",
"dependentCsp": "Azure",
"activePaymentModel": "Pay as you go",
"contractDetails": [
{
"startDate": "2024-01-01",
"endDate": "2025-06-15",
"discount": 20,
"paymentModel": "Committed Contract"
},
{
"startDate": "2024-02-01",
"endDate": "2025-07-15",
"discount": 20,
"paymentModel": "Committed Contract"
},
{
"startDate": "2024-03-16",
"endDate": null,
"discount": 0,
"paymentModel": "Pay as you go"
},
{
"startDate": "2025-03-25",
"endDate": null,
"discount": 0,
"paymentModel": "Pay as you go"
}
]
},
"createdAt": "2024-03-20T13:07:51Z",
"consumerOrgId": ""
},
{
"id": "c16a5b8d-0c80-4efd-97f8-a4370a97e901",
"vendorAccountName": "c16a5b8d-0c80-4efd-97f8-a4370a97e901",
"vendorAccountId": "c16a5b8d-0c80-4efd-97f8-a4370a97e901",
"vendorKey": "databricks",
"verification": {
"state": "error",
"message": "Failed to retrieve access token from Databricks. Status code: 400",
"lastVerificationAttemptedAt": "2024-03-20T13:11:31Z"
},
"authorization": {
"type": "databricks_user",
"clientId": "04540dca-4234-4d4c-87b5-bb24879a8f030",
"subscriptionModel": "Marketplace",
"dependentCsp": "Azure",
"activePaymentModel": "Pay as you go",
"contractDetails": [
{
"startDate": "2024-02-01",
"endDate": null,
"discount": 0,
"paymentModel": "Pay as you go"
},
{
"startDate": "2024-02-21",
"endDate": null,
"discount": 0,
"paymentModel": "Pay as you go"
}
]
},
"createdAt": "2024-03-20T13:02:12Z",
"consumerOrgId": ""
}
]
}

GET v3/vendors/databricks/accounts/

This API is used for retrieving all Databricks credentials for the given org.

Example Response 

{
"result": [
{
"id": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"vendorAccountName": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"vendorAccountId": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"vendorKey": "databricks",
"verification": {
"state": "verified",
"lastVerificationAttemptedAt": "2024-03-20T13:10:28Z"
},
"authorization": {
"type": "databricks_user",
"client_id": "b40563d6-d7de-4128-a542-c2f50c2a7ab8", "workspace": "apptio-yipit-pipeline.cloud.databricks.com", "warehouse_id": "f020534a981405ab",
"subscriptionModel": "Marketplace",
"dependentCsp": "Azure",
"activePaymentModel": "Pay as you go",
"contractDetails": [
{
"startDate": "2024-01-01",
"endDate": "2025-06-15",
"discount": 20,
"paymentModel": "Committed Contract"
},
{
"startDate": "2024-02-01",
"endDate": "2025-07-15",
"discount": 20,
"paymentModel": "Committed Contract"
},
{
"startDate": "2024-03-16",
"endDate": null,
"discount": 0,
"paymentModel": "Pay as you go"
},
{
"startDate": "2025-03-25",
"endDate": null,
"discount": 0,
"paymentModel": "Pay as you go"
}
]
},
"createdAt": "2024-03-20T13:07:51Z",
"consumerOrgId": ""
},
{
"id": "c16a5b8d-0c80-4efd-97f8-a4370a97e901",
"vendorAccountName": "c16a5b8d-0c80-4efd-97f8-a4370a97e901",
"vendorAccountId": "c16a5b8d-0c80-4efd-97f8-a4370a97e901",
"vendorKey": "databricks",
"verification": {
"state": "error",
"message": "Failed to retrieve access token from Databricks. Status code: 400",
"lastVerificationAttemptedAt": "2024-03-20T13:11:31Z"
},
"authorization": {
"type": "databricks_user",
"clientId": "04540dca-4234-4d4c-87b5-bb24879a8f030",
"subscriptionModel": "Marketplace",
"dependentCsp": "Azure",
"activePaymentModel": "Pay as you go",
"contractDetails": [
{
"startDate": "2024-02-01",
"endDate": null,
"discount": 0,
"paymentModel": "Pay as you go"
},
{
"startDate": "2024-02-21",
"endDate": null,
"discount": 0,
"paymentModel": "Pay as you go"
}
]
},
"createdAt": "2024-03-20T13:02:12Z",
"consumerOrgId": ""
}
]
}

DELETE v3/vendors/databricks/accounts/:accountId

This API is used for deleting the Databricks credential with the given ID.

Example Response 

{
"result": {
"message": "Successfully deleted",
"vendorKey": "databricks",
"vendorAccountId": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b"
}
}

POST v3/vendors/databricks/accounts/:accountId/verification

This API is used to verify the credential with the given ID.

Example Response 

{
"result": {
"id": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"vendorAccountName": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"vendorAccountId": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"vendorKey": "databricks",
"verification": {
"state": "verified",
"lastVerificationAttemptedAt": "2024-03-20T14:01:14Z"
},
"authorization": {
"type": "databricks_user",
"client_id": "b40563d6-d7de-4128-a542-c2f50c2a7ab8", "workspace": "apptio-yipit-pipeline.cloud.databricks.com", "warehouse_id": "f020534a981405ab",
"subscriptionModel": "Marketplace",
"dependentCsp": "Azure",
"activePaymentModel": "Pay as you go",
"contractDetails": [
{
"startDate": "2024-01-01",
"endDate": "2025-06-15",
"discount": 50,
"paymentModel": "Committed Contract"
},
{
"startDate": "2024-02-01",
"endDate": "2025-07-15",
"discount": 50,
"paymentModel": "Committed Contract"
}
]
},
"createdAt": "2024-03-20T13:55:57Z",
"consumerOrgId": ""
}
}

POST v3/vendors/databricks/accounts/:accountId/archive

This API is used to archive the Databricks account.

Example Response 

{
"result": {
"id": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"vendorAccountName": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"vendorAccountId": "c16a5b8d-0c80-4efd-97f8-a4370a97e90b",
"vendorKey": "databricks",
"verification": {
"state": "archived"
},
"meta": {},
"consumerOrgId": ""
}
}

GET v3/vendors/databricks/template

This API is used to get the template notebook.

Example Response 

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "b7df6f6c-b571-4cbf-8321-f04c0c3ad88f",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "%sql\n",
    "DESCRIBE system.billing.list_prices;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "051de69e-5044-471d-917c-86a1d410105a",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "%sql\n",
    "DESCRIBE system.billing.usage;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "e33e9e42-503e-42d5-a38b-4520f9781303",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "%sql\n",
    "CREATE CATALOG IF NOT EXISTS  billing_data;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "cb0436da-ee45-4f3d-96d6-abbebf582382",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "%sql\n",
    "CREATE SCHEMA IF NOT EXISTS  billing_data.billing_data_schema;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "6203679f-6a32-4c0b-b2cc-137f10d5a60b",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "%sql\n",
    "CREATE OR REPLACE TABLE billing_data.billing_data_schema.billing_materialized_view (\n",
    "  account_id\tstring,\n",
    "  workspace_id\tstring,\n",
    "  record_id\tstring,\n",
    "  sku_name\tstring,\n",
    "  cloud\tstring,\n",
    "  usage_start_time\ttimestamp,\n",
    "  usage_end_time\ttimestamp,\n",
    "  usage_date\tdate,\n",
    "  custom_tags\tmap<string,string>,\n",
    "  usage_unit\tstring,\n",
    "  usage_quantity\tdecimal(38,18),\n",
    "  usage_metadata\tstruct<cluster_id:string,job_id:string,warehouse_id:string,instance_pool_id:string,node_type:string,job_run_id:string,notebook_id:string,dlt_pipeline_id:string,endpoint_name:string,endpoint_id:string,dlt_update_id:string,dlt_maintenance_id:string>,\n",
    "  identity_metadata\tstruct<run_as:string>,\n",
    "  record_type\tstring,\n",
    "  ingestion_date\tdate,\n",
    "  billing_origin_product\tstring,\n",
    "  product_features\tstruct<jobs_tier:string,sql_tier:string,dlt_tier:string,is_serverless:boolean,is_photon:boolean,serving_type:string>,\n",
    "  usage_type\tstring,\n",
    "  report_period STRING,\n",
    "  unit_cost DECIMAL(38,18),\n",
    "  list_cost DECIMAL(38,18),\n",
    "  currency string,\n",
    "  price_start_time timestamp,\n",
    "  price_end_time timestamp\n",
    ")\n",
    "USING DELTA;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "cc59b3f7-14e7-4070-9260-31824949b7e8",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "%sql\n",
    "TRUNCATE billing_data.billing_data_schema.billing_materialized_view"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "eda22329-1288-4d1a-bfd9-70292a5d9054",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "%sql\n",
    "INSERT INTO billing_data.billing_data_schema.billing_materialized_view\n",
    "SELECT t1.*, DATE_FORMAT(usage_start_time, 'yyyy-MM') AS report_period, list_prices.pricing.default AS unit_cost, (t1.usage_quantity * list_prices.pricing.default) AS list_cost, list_prices.currency_code as currency, list_prices.price_start_time, list_prices.price_end_time\n",
    "FROM \n",
    "  system.billing.usage t1\n",
    "INNER JOIN \n",
    "  system.billing.list_prices list_prices \n",
    "ON\n",
    "  t1.cloud = list_prices.cloud AND\n",
    "  t1.sku_name = list_prices.sku_name AND\n",
    "  t1.usage_start_time >= list_prices.price_start_time AND\n",
    "  (t1.usage_end_time <= list_prices.price_end_time OR list_prices.price_end_time is null)"
   ]
  }
 ],
 "metadata": {
  "application/vnd.databricks.v1+notebook": {
   "dashboards": [],
   "environmentMetadata": null,
   "language": "python",
   "notebookMetadata": {
    "pythonIndentUnit": 4
   },
   "notebookName": "databricks_billing_details",
   "widgets": {}
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}