Databases
The Datalogz application uses OLTP and OLAP databases to optimize performance and scalability between the two workloads.
Database System Requirements
Option 1: Postgres Only
Azure Database for PostgreSQL (flexible server)
Minimum
Memory Optimized
Compute: 2 vCores, 16 GiB Memory, 3200 IOPS
Storage: 512 GB
Estimated Cost as of 2/1/2023: $500 / mo.
Recommended
Memory Optimized
Compute: 4 vCores, 32 GiB Memory, 6400 IOPS
Storage: 512 GB
Estimated Cost as of 2/1/2023: $850 / mo.
Total Estimated Cost: $500 - $850 / mo.
Option 2: Postgres + Snowflake
Postgres
Azure Database for PostgreSQL
Minimum/Recommended
Memory Optimized
Compute: 2 vCores, 8 GiB Memory, 3200 IOPS
Storage: 512 GB
Estimated Cost as of 2/1/2023: $200 / mo.
Snowflake
Minimum
Compute: X-Small Warehouse
Storage: Pay as you go
Estimated Cost as of 2/1/2023 assuming 10 users: $600 / mo.
Recommended
Compute: Small Warehouse
Storage: Pay as you go
Estimated Cost as of 2/1/2023 assuming 10 users: $1000 / mo.
Total Estimated Cost: $800 - $1200 / mo.
Object Creation
Option 1: Postgres Only
App DB (Azure Database for PostgreSQL) Name:
datalogz_bi
---- Run the following commands from the master (postgres) database
---- Run one line at a time
CREATE DATABASE datalogz_bi;
CREATE ROLE datalogz_gateway_admin WITH PASSWORD '<password>' LOGIN; -- Add the password!
GRANT ALL PRIVILEGES ON DATABASE datalogz_bi TO datalogz_gateway_admin WITH GRANT OPTION;
CREATE DATABASE airflow;
CREATE ROLE airflow WITH PASSWORD '<password>' LOGIN; -- Add the password!
GRANT ALL PRIVILEGES ON DATABASE airflow TO airflow;
---- Connect to the datalogz_bi database as datalogz_gateway_admin
---- Run the following commands from the datalogz_bi database
GRANT ALL PRIVILEGES ON SCHEMA public TO datalogz_gateway_admin;
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO datalogz_gateway_admin;Warehouse DB (Azure Database for PostgreSQL) Name:
datalogz_wh
---- Run the following commands from the master (postgres) database
-- Setup the BI Warehouse Database (OLAP - App Workflows)
CREATE DATABASE datalogz_wh;
CREATE ROLE datalogz_diagnostics_admin WITH PASSWORD '<password>' LOGIN; -- Add the password!
CREATE ROLE datalogz_gateway_admin WITH PASSWORD '<password>' LOGIN CREATEROLE; -- Add the password!
CREATE ROLE datalogz_gateway_user WITH PASSWORD '<password>' LOGIN; -- Add the password!
CREATE ROLE write_user_role;
CREATE ROLE read_user_role;
GRANT ALL ON DATABASE datalogz_wh TO datalogz_diagnostics_admin WITH GRANT OPTION;
GRANT ALL ON DATABASE datalogz_wh TO datalogz_gateway_admin WITH GRANT OPTION;
GRANT ALL ON DATABASE datalogz_wh TO write_user_role;
---- Run the following commands from the datalogz_wh database
CREATE SCHEMA powerbi_ingest;
CREATE SCHEMA powerbi_source;
CREATE SCHEMA tableau_ingest;
CREATE SCHEMA tableau_source;
CREATE SCHEMA biops_general;
CREATE SCHEMA biops_int;
CREATE SCHEMA biops_marts;
GRANT USAGE ON SCHEMA powerbi_ingest TO datalogz_diagnostics_admin;
GRANT USAGE ON SCHEMA powerbi_source TO datalogz_diagnostics_admin;
GRANT USAGE ON SCHEMA tableau_ingest TO datalogz_diagnostics_admin;
GRANT USAGE ON SCHEMA tableau_source TO datalogz_diagnostics_admin;
GRANT USAGE ON SCHEMA biops_general TO datalogz_diagnostics_admin;
GRANT USAGE ON SCHEMA biops_int TO datalogz_diagnostics_admin;
GRANT USAGE ON SCHEMA biops_marts TO datalogz_diagnostics_admin;
GRANT USAGE ON SCHEMA biops_general TO datalogz_gateway_admin;
GRANT USAGE ON SCHEMA biops_marts TO datalogz_gateway_admin;
GRANT USAGE ON SCHEMA biops_general TO read_user_role;
GRANT USAGE ON SCHEMA biops_marts TO read_user_role;
GRANT ALL ON SCHEMA biops_general TO write_user_role;
ALTER SCHEMA powerbi_ingest OWNER TO datalogz_diagnostics_admin;
ALTER SCHEMA powerbi_source OWNER TO datalogz_diagnostics_admin;
ALTER SCHEMA tableau_ingest OWNER TO datalogz_diagnostics_admin;
ALTER SCHEMA tableau_source OWNER TO datalogz_diagnostics_admin;
ALTER SCHEMA biops_general OWNER TO datalogz_diagnostics_admin;
GRANT read_user_role TO datalogz_gateway_user;
GRANT write_user_role TO datalogz_gateway_admin;
CREATE TABLE biops_general.entitlement(
account_id int,
environment_id int,
connector_id int,
account_connector_key varchar(128),
user_name varchar(256),
system_name varchar(256),
entitlement_type varchar(256),
entitlement_status varchar(256),
created_at timestamp default current_timestamp,
updated_at timestamp default current_timestamp
);
GRANT SELECT, INSERT, UPDATE ON ALL TABLES IN SCHEMA powerbi_ingest TO datalogz_diagnostics_admin;
GRANT SELECT, INSERT, UPDATE ON ALL TABLES IN SCHEMA powerbi_source TO datalogz_diagnostics_admin;
GRANT SELECT, INSERT, UPDATE ON ALL TABLES IN SCHEMA tableau_ingest TO datalogz_diagnostics_admin;
GRANT SELECT, INSERT, UPDATE ON ALL TABLES IN SCHEMA tableau_source TO datalogz_diagnostics_admin;
GRANT SELECT, INSERT, UPDATE ON ALL TABLES IN SCHEMA biops_general TO datalogz_diagnostics_admin;
GRANT SELECT, INSERT, UPDATE ON ALL TABLES IN SCHEMA powerbi_ingest TO datalogz_gateway_admin;
GRANT SELECT, INSERT, UPDATE ON ALL TABLES IN SCHEMA powerbi_source TO datalogz_gateway_admin;
GRANT SELECT, INSERT, UPDATE ON ALL TABLES IN SCHEMA tableau_ingest TO datalogz_gateway_admin;
GRANT SELECT, INSERT, UPDATE ON ALL TABLES IN SCHEMA tableau_source TO datalogz_gateway_admin;
GRANT SELECT, INSERT, UPDATE ON ALL TABLES IN SCHEMA biops_general TO datalogz_gateway_admin;
GRANT SELECT ON TABLE datalogz_wh.biops_general.entitlement TO read_user_role;
GRANT INSERT, SELECT, UPDATE ON TABLE datalogz_wh.biops_general.entitlement TO write_user_role;
CREATE EXTENSION IF NOT EXISTS FUZZYSTRMATCH WITH SCHEMA biops_marts;
CREATE EXTENSION IF NOT EXISTS plpgsql;
Option 2: Postgres + Snowflake
Postgres
App DB (Azure Database for PostgreSQL) Name:
datalogz_bi
-- Setup the Application Database (OLTP - App Workflows)
CREATE DATABASE datalogz_bi;
CREATE ROLE datalogz_gateway_admin WITH PASSWORD '<password>' LOGIN; -- Add the password!
GRANT ALL PRIVILEGES ON DATABASE datalogz_bi TO datalogz_gateway_db_admin WITH GRANT OPTION;
GRANT ALL PRIVILEGES ON SCHEMA public TO datalogz_gateway_db_admin;
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO datalogz_gateway_db_admin;
-- Setup the Airflow Database (OLTP - API Connector Workflows)
CREATE DATABASE airflow;
CREATE ROLE airflow WITH PASSWORD '<password>' LOGIN; -- Add the password!
GRANT ALL PRIVILEGES ON DATABASE airflow TO airflow;Snowflake
Warehouse DB (Snowflake) Name:
datalogz_wh
-- Setup the BI Warehouse Database (OLAP - App Workflows)
USE ROLE ACCOUNTADMIN;
CREATE WAREHOUSE IF NOT EXISTS DATALOGZ_BIOPS WITH WAREHOUSE_SIZE='X-SMALL';
USE ROLE SYSADMIN; -- whichever role you want to own this database
CREATE DATABASE DATALOGZ_WH;
CREATE SCHEMA IF NOT EXISTS POWERBI_INGEST;
CREATE SCHEMA IF NOT EXISTS POWERBI_SOURCE;
CREATE SCHEMA IF NOT EXISTS TABLEAU_INGEST;
CREATE SCHEMA IF NOT EXISTS TABLEAU_SOURCE;
CREATE SCHEMA IF NOT EXISTS BIOPS_GENERAL;
CREATE ROLE IF NOT EXISTS DATALOGZ_DIAGNOSTICS_ADMIN_ROLE;
CREATE ROLE IF NOT EXISTS DATALOGZ_GATEWAY_ADMIN_ROLE;
CREATE ROLE IF NOT EXISTS WRITE_USER_ROLE;
CREATE ROLE IF NOT EXISTS READ_USER_ROLE;
GRANT CREATE USER ON ACCOUNT TO ROLE WRITE_USER_ROLE;
GRANT ROLE USERADMIN TO ROLE WRITE_USER_ROLE;
GRANT ROLE WRITE_USER_ROLE TO ROLE DATALOGZ_GATEWAY_ADMIN_ROLE;
GRANT USAGE ON WAREHOUSE DATALOGZ_BIOPS TO ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE;
GRANT USAGE ON WAREHOUSE DATALOGZ_BIOPS TO ROLE DATALOGZ_GATEWAY_ADMIN_ROLE;
GRANT USAGE ON WAREHOUSE DATALOGZ_BIOPS TO ROLE USERADMIN;
GRANT USAGE ON WAREHOUSE DATALOGZ_BIOPS TO ROLE READ_USER_ROLE;
GRANT USAGE, OPERATE ON WAREHOUSE DATALOGZ_BIOPS TO ROLE WRITE_USER_ROLE;
GRANT USAGE ON DATABASE DATALOGZ_WH TO ROLE READ_USER_ROLE;
GRANT USAGE ON DATABASE DATALOGZ_WH TO ROLE WRITE_USER_ROLE;
GRANT ALL PRIVILEGES ON DATABASE DATALOGZ_WH TO ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE WITH GRANT OPTION;
GRANT ALL PRIVILEGES ON DATABASE DATALOGZ_WH TO ROLE DATALOGZ_GATEWAY_ADMIN_ROLE WITH GRANT OPTION;
GRANT USAGE ON SCHEMA BIOPS_MARTS TO ROLE READ_USER_ROLE;
GRANT USAGE ON SCHEMA BIOPS_GENERAL TO ROLE READ_USER_ROLE;
GRANT USAGE ON SCHEMA BIOPS_GENERAL TO ROLE WRITE_USER_ROLE;
GRANT ALL PRIVILEGES ON SCHEMA POWERBI_INGEST TO ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE;
GRANT ALL PRIVILEGES ON SCHEMA POWERBI_SOURCE TO ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE;
GRANT ALL PRIVILEGES ON SCHEMA TABLEAU_INGEST TO ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE;
GRANT ALL PRIVILEGES ON SCHEMA TABLEAU_SOURCE TO ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE;
GRANT ALL PRIVILEGES ON SCHEMA BIOPS_GENERAL TO ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE;
CREATE USER IF NOT EXISTS DATALOGZ_DIAGNOSTICS_ADMIN WITH PASSWORD = '<password>' DEFAULT_ROLE = DATALOGZ_DIAGNOSTICS_ADMIN_ROLE; -- Add the password!
CREATE USER IF NOT EXISTS DATALOGZ_GATEWAY_ADMIN WITH PASSWORD = '<password>' DEFAULT_ROLE = DATALOGZ_GATEWAY_ADMIN_ROLE; -- Add the password!
GRANT OWNERSHIP ON ROLE READ_USER_ROLE TO ROLE DATALOGZ_GATEWAY_ADMIN_ROLE REVOKE CURRENT GRANTS;
GRANT OWNERSHIP ON ROLE WRITE_USER_ROLE TO ROLE DATALOGZ_GATEWAY_ADMIN_ROLE REVOKE CURRENT GRANTS;
GRANT SELECT, INSERT, UPDATE ON FUTURE TABLES IN SCHEMA POWERBI_INGEST TO ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE;
GRANT SELECT, INSERT, UPDATE ON FUTURE TABLES IN SCHEMA POWERBI_SOURCE TO ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE;
GRANT SELECT, INSERT, UPDATE ON FUTURE TABLES IN SCHEMA TABLEAU_INGEST TO ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE;
GRANT SELECT, INSERT, UPDATE ON FUTURE TABLES IN SCHEMA TABLEAU_SOURCE TO ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE;
GRANT SELECT, INSERT, UPDATE ON FUTURE TABLES IN SCHEMA BIOPS_GENERAL TO ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE;
GRANT SELECT, INSERT, UPDATE ON FUTURE TABLES IN SCHEMA BIOPS_GENERAL TO ROLE DATALOGZ_GATEWAY_ADMIN_ROLE;
GRANT ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE TO USER DATALOGZ_DIAGNOSTICS_ADMIN;
GRANT ROLE DATALOGZ_GATEWAY_ADMIN_ROLE TO USER DATALOGZ_GATEWAY_ADMIN;
USE ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE;
CREATE SCHEMA IF NOT EXISTS DATALOGZ_WH.POWERBI_INGEST;
CREATE SCHEMA IF NOT EXISTS DATALOGZ_WH.POWERBI_SOURCE;
CREATE SCHEMA IF NOT EXISTS DATALOGZ_WH.TABLEAU_INGEST;
CREATE SCHEMA IF NOT EXISTS DATALOGZ_WH.TABLEAU_SOURCE;
CREATE SCHEMA IF NOT EXISTS DATALOGZ_WH.BIOPS_GENERAL;
CREATE OR REPLACE TABLE DATALOGZ_WH.BIOPS_GENERAL.ENTITLEMENT(
account_id int,
environment_id int,
connector_id int,
user_name varchar(256),
system_name varchar(256),
entitlement_type varchar(256),
entitlement_status varchar(256),
created_at timestamp default current_timestamp(),
updated_at timestamp default current_timestamp()
);
GRANT INSERT, SELECT, UPDATE ON TABLE DATALOGZ_WH.BIOPS_GENERAL.ENTITLEMENT TO ROLE WRITE_USER_ROLE;
CREATE FILE FORMAT IF NOT EXISTS json_file_format
TYPE = 'json' STRIP_OUTER_ARRAY = true STRIP_NULL_VALUES = true;
USE ROLE ACCOUNTADMIN;
CREATE OR REPLACE STORAGE INTEGRATION <name_of_storage_integration> -- Add a name!
TYPE = EXTERNAL_STAGE
STORAGE_PROVIDER = 'AZURE'
ENABLED = TRUE
AZURE_TENANT_ID = '' -- Add the Tenant ID!
STORAGE_ALLOWED_LOCATIONS = ('azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_workbooks/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_workbooks_connections/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_workbooks_extensions_analytics_connections/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_workbooks_preview_image/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_workbooks_revisions/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_workbooks_usage/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_workbooks_search_usage/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_workbooks_views/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_metrics/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_workbooks_scan_result/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/scan_results_datasources/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/scan_results_datasources_tables/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/scan_results_datasources_tables_columns/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_sites_views/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_sites_views_search_usage/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_permissions/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_permissions_groups_capabilities/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/datasources/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/datasources_connections/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/datasources_revisions/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_datasources_certifications/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/datasources_usage/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/datasources_search_usage/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/flows/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/flows_runs/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/flows_permissions/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/flows_permissions_groups_capabilities/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/flows_connections/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/flows_usage/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/flows_search_usage/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/groups/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/groups_users/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/groups_users_favorites/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/extensions_analytics_connections/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/connected_apps/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/custom_views/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/data_alerts/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/data_alerts_details/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/schedules/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/schedules_extracts/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_tables/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_tables_columns/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_tables_permissions/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_tables_certifications/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_tables_search_usage/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_databases/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_databases_certifications/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_databases_permissions/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/jobs/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/recommendations/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/settings_mobile_security_settings/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/settings_user_notifications/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/sites_content_recently_viewed/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/sites_embedding_settings/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/subscriptions/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/tasks_extract_refreshes/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/tasks_linked/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/tasks_run_flows/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/virtual_connections/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_virtual_connections_certifications/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/saved_issues/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_workbooks/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_workbooks_connections/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_workbooks_extensions_analytics_connections/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_workbooks_preview_image/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_workbooks_revisions/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_workbooks_usage/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_workbooks_search_usage/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_workbooks_views/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_metrics/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_workbooks_scan_result/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/scan_results_datasources/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/scan_results_datasources_tables/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/scan_results_datasources_tables_columns/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_sites_views/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_sites_views_search_usage/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_permissions/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_permissions_groups_capabilities/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/datasources/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/datasources_connections/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/datasources_revisions/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_datasources_certifications/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/datasources_usage/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/datasources_search_usage/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/flows/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/flows_runs/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/flows_permissions/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/flows_permissions_groups_capabilities/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/flows_connections/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/flows_usage/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/flows_search_usage/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/groups/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/groups_users/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/groups_users_favorites/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/extensions_analytics_connections/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/connected_apps/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/custom_views/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/data_alerts/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/data_alerts_details/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/schedules/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/schedules_extracts/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_tables/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_tables_columns/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_tables_permissions/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_tables_certifications/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_tables_search_usage/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_databases/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_databases_certifications/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_databases_permissions/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/jobs/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/recommendations/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/settings_mobile_security_settings/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/settings_user_notifications/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/sites_content_recently_viewed/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/sites_embedding_settings/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/subscriptions/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/tasks_extract_refreshes/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/tasks_linked/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/tasks_run_flows/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/virtual_connections/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_virtual_connections_certifications/',
'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/saved_issues/',
);
SHOW INTEGRATIONS;
DESC STORAGE INTEGRATION <name_of_storage_integration>;
SELECT SYSTEM$GET_SNOWFLAKE_PLATFORM_INFO();
-- add network rules for each of the subnets displayed using the Azure CLI
-- az storage account network-rule add --account-name <name_of_sa> --resource-group <name_of_rg> --subnet "<name_of_subnet>"
-- then proceed below
GRANT CREATE STAGE ON SCHEMA POWERBI_INGEST TO ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE;
GRANT CREATE STAGE ON SCHEMA TABLEAU_INGEST TO ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE;
GRANT USAGE ON INTEGRATION <name_of_storage_integration> TO ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE;
USE ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE;
GRANT INSERT, SELECT, UPDATE ON TABLE DATALOGZ_WH.BIOPS_GENERAL.ENTITLEMENT TO ROLE WRITE_USER_ROLE;
CREATE FILE FORMAT IF NOT EXISTS json_file_format
TYPE = 'json' STRIP_OUTER_ARRAY = true STRIP_NULL_VALUES = true;
USE ROLE DATALOGZ_DIAGNOSTICS_ADMIN_ROLE;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_projects storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_projects_workbooks storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_workbooks/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_projects_workbooks_connections storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_workbooks_connections/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_projects_workbooks_extensions_analytics_connections storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_workbooks_extensions_analytics_connections/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_projects_workbooks_preview_image storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_workbooks_preview_image/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_projects_workbooks_revisions storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_workbooks_revisions/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_projects_workbooks_usage storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_workbooks_usage/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_projects_workbooks_search_usage storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_workbooks_search_usage/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_projects_workbooks_views storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_workbooks_views/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_projects_metrics storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_metrics/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_projects_workbooks_scan_result storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_workbooks_scan_result/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_scan_results_datasources storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/scan_results_datasources/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_scan_results_datasources_tables storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/scan_results_datasources_tables/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_scan_results_datasources_tables_columns storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/scan_results_datasources_tables_columns/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_projects_sites_views storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_sites_views/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_projects_sites_views_search_usage storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_sites_views_search_usage/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_projects_permissions storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_permissions/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_projects_permissions_groups_capabilities storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/projects_permissions_groups_capabilities/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_datasources storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/datasources/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_datasources_connections storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/datasources_connections/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_datasources_revisions storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/datasources_revisions/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_metadata_datasources_certifications storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_datasources_certifications/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_datasources_usage storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/datasources_usage/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_datasources_search_usage storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/datasources_search_usage/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_flows storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/flows/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_flows_runs storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/flows_runs/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_flows_permissions storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/flows_permissions/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_flows_permissions_groups_capabilities storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/flows_permissions_groups_capabilities/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_flows_connections storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/flows_connections/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_flows_usage storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/flows_usage/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_flows_search_usage storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/flows_search_usage/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_groups storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/groups/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_groups_users storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/groups_users/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_groups_users_favorites storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/groups_users_favorites/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_extensions_analytics_connections storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/extensions_analytics_connections/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_connected_apps storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/connected_apps/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_custom_views storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/custom_views/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_data_alerts storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/data_alerts/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_data_alerts_details storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/data_alerts_details/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_schedules storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/schedules/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_schedules_extracts storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/schedules_extracts/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_metadata_tables storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_tables/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_metadata_tables_columns storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_tables_columns/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_metadata_tables_permissions storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_tables_permissions/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_metadata_tables_certifications storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_tables_certifications/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_metadata_tables_search_usage storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_tables_search_usage/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_metadata_databases storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_databases/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_metadata_databases_certifications storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_databases_certifications/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_metadata_databases_permissions storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_databases_permissions/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_jobs storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/jobs/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_recommendations storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/recommendations/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_settings_mobile_security_settings storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/settings_mobile_security_settings/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_settings_user_notifications storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/settings_user_notifications/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_sites_content_recently_viewed storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/sites_content_recently_viewed/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_sites_embedding_settings storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/sites_embedding_settings/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_subscriptions storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/subscriptions/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_tasks_extract_refreshes storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/tasks_extract_refreshes/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_tasks_linked storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/tasks_linked/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_tasks_run_flows storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/tasks_run_flows/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_virtual_connections storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/virtual_connections/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_metadata_virtual_connections_certifications storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/metadata_virtual_connections_certifications/' file_format = json_file_format;
create stage if not exists datalogz_wh.powerbi_ingest.powerbi_saved_issues storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/powerbi/admin/saved_issues/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_projects storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_projects_workbooks storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_workbooks/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_projects_workbooks_connections storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_workbooks_connections/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_projects_workbooks_extensions_analytics_connections storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_workbooks_extensions_analytics_connections/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_projects_workbooks_preview_image storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_workbooks_preview_image/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_projects_workbooks_revisions storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_workbooks_revisions/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_projects_workbooks_usage storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_workbooks_usage/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_projects_workbooks_search_usage storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_workbooks_search_usage/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_projects_workbooks_views storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_workbooks_views/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_projects_metrics storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_metrics/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_projects_workbooks_scan_result storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_workbooks_scan_result/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_scan_results_datasources storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/scan_results_datasources/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_scan_results_datasources_tables storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/scan_results_datasources_tables/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_scan_results_datasources_tables_columns storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/scan_results_datasources_tables_columns/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_projects_sites_views storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_sites_views/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_projects_sites_views_search_usage storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_sites_views_search_usage/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_projects_permissions storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_permissions/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_projects_permissions_groups_capabilities storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/projects_permissions_groups_capabilities/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_datasources storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/datasources/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_datasources_connections storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/datasources_connections/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_datasources_revisions storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/datasources_revisions/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_metadata_datasources_certifications storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_datasources_certifications/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_datasources_usage storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/datasources_usage/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_datasources_search_usage storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/datasources_search_usage/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_flows storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/flows/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_flows_runs storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/flows_runs/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_flows_permissions storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/flows_permissions/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_flows_permissions_groups_capabilities storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/flows_permissions_groups_capabilities/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_flows_connections storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/flows_connections/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_flows_usage storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/flows_usage/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_flows_search_usage storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/flows_search_usage/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_groups storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/groups/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_groups_users storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/groups_users/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_groups_users_favorites storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/groups_users_favorites/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_extensions_analytics_connections storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/extensions_analytics_connections/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_connected_apps storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/connected_apps/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_custom_views storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/custom_views/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_data_alerts storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/data_alerts/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_data_alerts_details storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/data_alerts_details/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_schedules storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/schedules/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_schedules_extracts storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/schedules_extracts/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_metadata_tables storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_tables/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_metadata_tables_columns storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_tables_columns/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_metadata_tables_permissions storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_tables_permissions/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_metadata_tables_certifications storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_tables_certifications/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_metadata_tables_search_usage storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_tables_search_usage/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_metadata_databases storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_databases/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_metadata_databases_certifications storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_databases_certifications/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_metadata_databases_permissions storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_databases_permissions/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_jobs storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/jobs/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_recommendations storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/recommendations/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_settings_mobile_security_settings storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/settings_mobile_security_settings/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_settings_user_notifications storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/settings_user_notifications/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_sites_content_recently_viewed storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/sites_content_recently_viewed/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_sites_embedding_settings storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/sites_embedding_settings/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_subscriptions storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/subscriptions/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_tasks_extract_refreshes storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/tasks_extract_refreshes/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_tasks_linked storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/tasks_linked/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_tasks_run_flows storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/tasks_run_flows/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_virtual_connections storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/virtual_connections/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_metadata_virtual_connections_certifications storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/metadata_virtual_connections_certifications/' file_format = json_file_format;
create stage if not exists datalogz_wh.tableau_ingest.tableau_saved_issues storage_integration = datalogz_wh_saas_prod_int url = 'azure://<blob_storage_name>.blob.core.windows.net/<container_name>/data/tableau/admin/saved_issues/' file_format = json_file_format;
Last updated