From 8847e969df86663f487d1959bf300c9e440be805 Mon Sep 17 00:00:00 2001 From: Jamie Davidson Date: Mon, 6 Jan 2025 14:35:19 -0800 Subject: [PATCH] wip --- examples/lookml_migrator.py | 201 ++++++++++++++++++ examples/resources/daily_funnel.view.lkml | 56 +++++ .../resources/distribution_centers.view.lkml | 9 + examples/resources/event_sessions.view.lkml | 44 ++++ examples/resources/events.explore.lkml | 8 + examples/resources/events.view.lkml | 44 ++++ .../resources/filtered_lookml_dt.view.lkml | 22 ++ examples/resources/inventory_items.view.lkml | 19 ++ examples/resources/lifetime_brand.view.lkml | 7 + .../lifetime_product_category.view.lkml | 7 + examples/resources/manifest.lkml | 12 ++ examples/resources/order_items.explore.lkml | 18 ++ examples/resources/order_items.view.lkml | 17 ++ examples/resources/probability_male.view.lkml | 15 ++ examples/resources/products.view.lkml | 18 ++ .../thelook_web_analytics.model.lkml | 17 ++ examples/resources/user_day_stats.view.lkml | 46 ++++ .../user_event_attribution.view.lkml | 35 +++ examples/resources/user_joins.explore.lkml | 16 ++ examples/resources/user_order_facts.view.lkml | 18 ++ .../resources/user_order_sequence.view.lkml | 16 ++ examples/resources/users.explore.lkml | 7 + examples/resources/users.view.lkml | 24 +++ examples/resources/zip_demographics.view.lkml | 33 +++ examples/test_lookml_migrator.py | 25 +++ 25 files changed, 734 insertions(+) create mode 100644 examples/lookml_migrator.py create mode 100644 examples/resources/daily_funnel.view.lkml create mode 100644 examples/resources/distribution_centers.view.lkml create mode 100644 examples/resources/event_sessions.view.lkml create mode 100644 examples/resources/events.explore.lkml create mode 100644 examples/resources/events.view.lkml create mode 100644 examples/resources/filtered_lookml_dt.view.lkml create mode 100644 examples/resources/inventory_items.view.lkml create mode 100644 examples/resources/lifetime_brand.view.lkml create mode 100644 examples/resources/lifetime_product_category.view.lkml create mode 100644 examples/resources/manifest.lkml create mode 100644 examples/resources/order_items.explore.lkml create mode 100644 examples/resources/order_items.view.lkml create mode 100644 examples/resources/probability_male.view.lkml create mode 100644 examples/resources/products.view.lkml create mode 100644 examples/resources/thelook_web_analytics.model.lkml create mode 100644 examples/resources/user_day_stats.view.lkml create mode 100644 examples/resources/user_event_attribution.view.lkml create mode 100644 examples/resources/user_joins.explore.lkml create mode 100644 examples/resources/user_order_facts.view.lkml create mode 100644 examples/resources/user_order_sequence.view.lkml create mode 100644 examples/resources/users.explore.lkml create mode 100644 examples/resources/users.view.lkml create mode 100644 examples/resources/zip_demographics.view.lkml create mode 100644 examples/test_lookml_migrator.py diff --git a/examples/lookml_migrator.py b/examples/lookml_migrator.py new file mode 100644 index 0000000..c0b012c --- /dev/null +++ b/examples/lookml_migrator.py @@ -0,0 +1,201 @@ +import requests +import os +import base64 +import lkml +import sys +import argparse +import pdb +from omni_python_sdk import OmniAPI + + +# Fetch the token if available +GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") + +# Set up headers with conditional authentication +headers = { + "Accept": "application/vnd.github.v3+json" +} +if GITHUB_TOKEN: + headers["Authorization"] = f"token {GITHUB_TOKEN}" + +def base_url(repo: str): + return f"https://api.github.com/repos/{repo}" + +def find_lkml_files_in_repo(repo: str, branch: str = "main"): + lkml_files = [] + url = f"{base_url(repo)}/git/trees/{branch}?recursive=1" # Adjust 'main' if you’re using a different branch + + response = requests.get(url, headers=headers) + response.raise_for_status() + + # Search for .lkml files in the tree response + files = response.json().get("tree", []) + for file in files: + if file["path"].endswith(".lkml"): + lkml_files.append(file["path"]) + + return lkml_files + +def get_file_content(repo: str, file_path: str): + url = f"{base_url(repo)}/contents/{file_path}" + response = requests.get(url, headers=headers) + response.raise_for_status() + + content = response.json().get("content") + if content: + # Decode the Base64 content + decoded_content = base64.b64decode(content).decode("utf-8") + return decoded_content + return None + +def parse_lkml_content(content): + parsed = lkml.load(content) + return parsed + +def parse_lookml(repo: str, branch: str): + lkml_files = find_lkml_files_in_repo(repo, branch) + parsed_objects = [] + + for file_path in lkml_files: + content = get_file_content(repo, file_path) + if content: + parsed = parse_lkml_content(content) + parsed_objects.append(parsed) + + # Continue with mapping and creating objects as needed + return parsed_objects + +def map_lookml_model_to_omni(parsed_model): + # Map LookML model to Omni model + pass + + +# explore_source: identifier { +# bind_all_filters: yes +# column: identifier { +# field: field_name +# } +# derived_column: identifier { +# sql: SQL expression ;; +# } +# expression_custom_filter: [custom filter expression] +# filters: [field_name_1: "string", field_name_2: "string", ...] +# limit: number +# sorts: [field_name_1: asc | desc, field_name_2: asc | desc, ...] +# timezone: "string" +# } +def map_lookml_query_to_omni(parsed_query): + omni_query = { + "topic": parsed_query["explore_source"]["name"] + } + omni_query["fields"]= {} + for column in parsed_query["columns"]: + if "field" in column: + omni_query["fields"][column["field"]] = column["name"] + else: + omni_query["fields"][column["name"]] = column["name"] + if "derived_columns" in parsed_query: + for derived_column in parsed_query["derived_columns"]: + # todo map derived column to omni + omni_query["fields"][derived_column["name"]] = derived_column["sql"] + if "filters_all" in parsed_query: + pass + if "persist_for" in parsed_query: + pass + if "bind_filters__all" in parsed_query: + pass + if "limit" in parsed_query: + pass + if "sorts" in parsed_query: + pass + if "timezone" in parsed_query: + pass + + +def map_lookml_view_to_omni(parsed_view): + omni_view = { + "name": parsed_view["name"] + } + + if "sql_table_name" in parsed_view: + # if schema is present + if parsed_view["sql_table_name"].count(".") > 0: + omni_view["schema"] = parsed_view["sql_table_name"].split(".")[0] + omni_view["table"] = parsed_view["sql_table_name"].split(".")[1] + else: + omni_view["table"] = parsed_view["sql_table_name"] + elif "derived_table" in parsed_view: + if "sql" in parsed_view["derived_table"]: + omni_view["sql"] = parsed_view["derived_table"]["sql"] + elif "explore_source" in parsed_view["derived_table"]: + omni_view["query"] = map_lookml_query_to_omni(parsed_view["derived_table"]["explore_source"]) + else: + # Handle derived_table: sql_trigger + pass + + # Map LookML view to Omni view + if "drill_fields" in parsed_view: + omni_view["drill_fields"] = parsed_view["drill_fields"] + if "label" in parsed_view: + omni_view["label"] = parsed_view["label"] + if "required_access_grants" in parsed_view: + # TODO map required access grants + # omni_view["required_access_grants"] = parsed_view["required_access_grants"] + pass + + # Map fields + if "dimensions" in parsed_view: + omni_view["dimensions"] = [map_lookml_dimension_to_omni(dim) for dim in parsed_view["dimensions"]] + + if "dimension_groups" in parsed_view: + dim_groups = [map_lookml_dimension_group_to_omni(dim_group) for dim_group in parsed_view["dimension_groups"]] + + if "dimensions" in omni_view: + omni_view["dimensions"] += dim_groups + else: + omni_view["dimensions"] = dim_groups + + if "measures" in parsed_view: + omni_view["measures"] = [map_lookml_measure_to_omni(measure) for measure in parsed_view["measures"]] + return omni_view + +def map_lookml_dimension_to_omni(parsed_field): + # Map LookML dimension to Omni dimension + pass + +def map_lookml_measure_to_omni(parsed_field): + # Map LookML measure to Omni measure + pass + +def map_lookml_dimension_group_to_omni(parsed_field): + # Map LookML dimension group to Omni date field + pass + +def map_lookml_explore_to_omni(parsed_explore): + # Map LookML explore to Omni topic + pass + +def migrate_lookml(repo: str, branch: str, client: OmniAPI, connection_id: str): + parsed_objects = parse_lookml(repo, branch) + # migrate lookml views + + views = [view for obj in parsed_objects if "views" in obj for view in obj["views"]] + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="LookML Migrator") + + # Define named arguments with the -- prefix + parser.add_argument("--github_repo", type=str, help="LookML GitHub repository", required=True) + parser.add_argument("--branch", type=str, help="Branch name", default="main") + parser.add_argument("--omni_api_key", type=str, help="Omni API key", required=True) + parser.add_argument("--omni_base_url", type=str, help="Omni base URL", required=True) + parser.add_argument("--omni_connection_id", type=str, help="Omni Connection ID", required=True) + + args = parser.parse_args() + + # Initialize the OmniAPI client + client = OmniAPI(args.omni_api_key, args.omni_base_url) + + migrate_lookml(args.github_repo, args.branch, client, args.omni_connection_id,) \ No newline at end of file diff --git a/examples/resources/daily_funnel.view.lkml b/examples/resources/daily_funnel.view.lkml new file mode 100644 index 0000000..5588eb1 --- /dev/null +++ b/examples/resources/daily_funnel.view.lkml @@ -0,0 +1,56 @@ +include: "order_items.explore" +include: "users.explore" + +explore: daily_funnel { + join: funnel_orders {relationship:one_to_one + sql_on: ${funnel_orders.order_date} = ${daily_funnel.date} ;;} + join: funnel_signups {relationship:one_to_one + sql_on: ${funnel_signups.signup_date} = ${daily_funnel.date} ;;} +} + + +# Start with a Date table +view: daily_funnel { + derived_table: { + sql: + SELECT * FROM ( + SELECT + DATE_ADD('2010-01-01', INTERVAL num DAY) as d + FROM UNNEST(GENERATE_ARRAY(1,10000)) num + ) + WHERE {% condition daily_funnel.filter_date %} + TIMESTAMP(d) + {% endcondition %} + ;; + } + filter: filter_date {type:date datatype:date} + dimension: date {can_filter:no sql: ${TABLE}.d ;;} +} + +# Join orders by day +view: funnel_orders { + derived_table: { + explore_source: order_items { + column: order_date {field:order_items.created_date} + column: order_count {field:order_items.order_count} + bind_filters: {from_field: daily_funnel.filter_date + to_field: order_items.created_date } + } + } + dimension: order_date {} + dimension: order_count {type: number} +} + +# Join user signups. +view: funnel_signups { + derived_table: { + explore_source: users { + column: signup_date {field:users.created_date} + column: signup_count {field:users.count} + bind_filters: {from_field:daily_funnel.filter_date + to_field:users.created_date} + } + } + dimension: signup_date {} + dimension: signup_count {type: number} +} diff --git a/examples/resources/distribution_centers.view.lkml b/examples/resources/distribution_centers.view.lkml new file mode 100644 index 0000000..c25a564 --- /dev/null +++ b/examples/resources/distribution_centers.view.lkml @@ -0,0 +1,9 @@ +view: distribution_centers { + sql_table_name: thelook_web_analytics.distribution_centers ;; + + dimension: id {primary_key:yes type:number} + dimension: latitude {type:number} + dimension: longitude {type:number} + dimension: name {} + measure: count {type:count drill_fields:[id, name, products.count]} +} diff --git a/examples/resources/event_sessions.view.lkml b/examples/resources/event_sessions.view.lkml new file mode 100644 index 0000000..33ad2df --- /dev/null +++ b/examples/resources/event_sessions.view.lkml @@ -0,0 +1,44 @@ +include: "products.view" +include: "users.view" +include: "events.explore" + +view: event_sessions { + derived_table: { + #persist_for: "2 hours" + explore_source: events { + column: session_id { field: events.session_id } + column: event_types { field: events.event_types } + column: session_time { field: events.minimum_time } + column: session_end_time {field: events.max_time} + column: ip_addresses {field: events.ip_addresses } + column: user_id {field: events.first_user_id } + column: product_ids_visited {field: events.product_ids_visited} + derived_column: session_sequence { + sql: ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY session_time) ;; + } + } + } + dimension: session_id {primary_key:yes} + dimension: event_types {} + dimension_group: session {type:time sql: ${TABLE}.session_time ;;} + dimension: session_end_time {hidden:yes} + dimension: ip_addresses {} + dimension: user_id {} + dimension: product_ids_visited {} + dimension: session_sequence {type:number} + dimension: session_length {type:number + sql: TIMESTAMP_DIFF(${session_end_time},${session_raw}, SECOND) ;;} + dimension: session_length_tiered {type:tier tiers: [0,60,120] sql: ${session_length} ;;} + + measure: count_sessions {type:count drill_fields:[session*]} + measure: count_sessions_with_cart {type:count drill_fields:[session*] + filters: {field:event_types value:"%Cart%"}} + measure: count_sessions_with_purchases {type:count drill_fields: [session*] + filters: {field:event_types value:"%Purchase%"}} + + set: session{ fields:[session_time, session_id, user_id, event_types]} +} + +view: id { + dimension: id {sql: ${TABLE} ;;} +} diff --git a/examples/resources/events.explore.lkml b/examples/resources/events.explore.lkml new file mode 100644 index 0000000..63708c8 --- /dev/null +++ b/examples/resources/events.explore.lkml @@ -0,0 +1,8 @@ +include: "users.view" +include: "events.view" +include: "user_joins.explore" + +explore: events { + extends: [user_joins] + join: users {relationship:many_to_one sql_on: ${events.user_id} = ${users.id} ;;} +} diff --git a/examples/resources/events.view.lkml b/examples/resources/events.view.lkml new file mode 100644 index 0000000..203886d --- /dev/null +++ b/examples/resources/events.view.lkml @@ -0,0 +1,44 @@ +view: events { + sql_table_name: thelook_web_analytics.events ;; + + dimension: id {primary_key:yes type:number} + dimension: browser {} + dimension: city {} + dimension: country {} + dimension_group: created {type:time sql: ${TABLE}.created_at ;;} + dimension: event_type {} + dimension: ip_address {} + dimension: latitude {type:number} + dimension: longitude {type:number} + dimension: os {} + dimension: sequence_number {type:number} + dimension: session_id {} + dimension: state {} + dimension: traffic_source {} + dimension: uri {} + dimension: user_id {type:number sql: CAST(REGEXP_EXTRACT(${TABLE}.user_id, r'\d+') AS INT64) ;;} + dimension: zip {} + + measure: count {type:count drill_fields:[id, users.last_name, users.id, users.first_name]} + + + # + # Sessionization Fields. + # + measure: minimum_time {sql: MIN(${created_raw}) ;;} + measure: max_time {sql: MAX(${created_raw}) ;;} + measure: ip_addresses {sql: ARRAY_TO_STRING(ARRAY_AGG(DISTINCT ${ip_address}),'|') ;;} + measure: event_types {sql: ARRAY_TO_STRING(ARRAY_AGG(DISTINCT ${event_type}),'|') ;;} + + # code defensivly. Should only have one user_id on a session, but just + # in case, we'll take the first one we see. + measure: first_user_id {type:min sql: ${user_id} ;;} + + # parse the product_id out of the urls visited and return an array of them. + measure: product_ids_visited { + sql: ARRAY_AGG(DISTINCT + CAST(REGEXP_EXTRACT(${uri}, r'/product/(\d+)') AS INT64) + IGNORE NULLS) ;; + } + +} diff --git a/examples/resources/filtered_lookml_dt.view.lkml b/examples/resources/filtered_lookml_dt.view.lkml new file mode 100644 index 0000000..728ed61 --- /dev/null +++ b/examples/resources/filtered_lookml_dt.view.lkml @@ -0,0 +1,22 @@ +include: "users.explore.lkml" + +explore: filtered_lookml_dt {} + +view: filtered_lookml_dt { + derived_table: { + explore_source: users { + column: age {field: users.age} + column: people {field: users.count} + bind_filters: { + to_field: users.created_date + from_field: filtered_lookml_dt.filter_date + } + } + } + + filter: filter_date { + type: date + } + dimension: age {type: number} + dimension: people {type: number} +} diff --git a/examples/resources/inventory_items.view.lkml b/examples/resources/inventory_items.view.lkml new file mode 100644 index 0000000..15eac0f --- /dev/null +++ b/examples/resources/inventory_items.view.lkml @@ -0,0 +1,19 @@ +view: inventory_items { + sql_table_name: thelook_web_analytics.inventory_items ;; + + dimension: id {primary_key:yes type:number} + dimension: cost {type:number} + dimension: created_at {} + dimension: product_brand {} + dimension: product_category {} + dimension: product_department {} + dimension: product_distribution_center_id {type:number} + dimension: product_id {type:number} + dimension: product_name {} + dimension: product_retail_price {type:number} + dimension: product_sku {} + dimension: sold_at {} + + measure: count {type:count + drill_fields:[id, product_name, products.name, products.id, order_items.count]} +} diff --git a/examples/resources/lifetime_brand.view.lkml b/examples/resources/lifetime_brand.view.lkml new file mode 100644 index 0000000..e7f8bcc --- /dev/null +++ b/examples/resources/lifetime_brand.view.lkml @@ -0,0 +1,7 @@ +view: lifetime_brand { + dimension: ever_purchased_brand { + suggest_explore: products + suggest_dimension: products.brand + sql: ${TABLE} ;; + } +} diff --git a/examples/resources/lifetime_product_category.view.lkml b/examples/resources/lifetime_product_category.view.lkml new file mode 100644 index 0000000..9a435f5 --- /dev/null +++ b/examples/resources/lifetime_product_category.view.lkml @@ -0,0 +1,7 @@ +view: lifetime_product_category { + dimension: ever_purchased_in_category { + suggest_explore: products + suggest_dimension: products.category + sql: ${TABLE} ;; + } +} diff --git a/examples/resources/manifest.lkml b/examples/resources/manifest.lkml new file mode 100644 index 0000000..48082a7 --- /dev/null +++ b/examples/resources/manifest.lkml @@ -0,0 +1,12 @@ +project_name: "bq_thelook" + +# Use local_dependency: To enable referencing of another project +# on this instance with include: statements + +local_dependency: { + project: "acs" +} + +local_dependency: { + project: "name_game2" +} diff --git a/examples/resources/order_items.explore.lkml b/examples/resources/order_items.explore.lkml new file mode 100644 index 0000000..498b4d8 --- /dev/null +++ b/examples/resources/order_items.explore.lkml @@ -0,0 +1,18 @@ +include: "users.view" +include: "inventory_items.view" +include: "products.view" +include: "distribution_centers.view" +include: "user_order_sequence.view" +include: "order_items.view" +include: "user_joins.explore" + + +explore: order_items { + extends: [user_joins] + join: users {relationship:many_to_one sql_on: ${order_items.user_id} = ${users.id} ;;} + join: inventory_items {relationship:many_to_one sql_on: ${order_items.inventory_item_id} = ${inventory_items.id} ;;} + join: products {relationship:many_to_one sql_on: ${inventory_items.product_id} = ${products.id} ;;} + join: distribution_centers {relationship:many_to_one sql_on: ${products.distribution_center_id} = ${distribution_centers.id} ;;} + join: user_order_sequence {view_label:"Order Items" relationship: many_to_one + sql_on: ${user_order_sequence.order_id} = ${order_items.order_id} ;;} +} diff --git a/examples/resources/order_items.view.lkml b/examples/resources/order_items.view.lkml new file mode 100644 index 0000000..920053f --- /dev/null +++ b/examples/resources/order_items.view.lkml @@ -0,0 +1,17 @@ +view: order_items { + sql_table_name: thelook_web_analytics.order_items ;; + + dimension: id {primary_key:yes type:number} + dimension_group: created {type:time sql: TIMESTAMP(${TABLE}.created_at) ;;} + dimension: inventory_item_id {type:number} + dimension: order_id {type:number} + dimension: sale_price {type: number} + dimension: status {} + dimension: user_id {type: number} + + measure: orders_per_capita {type:number + sql: ${order_count}/NULLIF(${zip_demographics.total_population}, 0) ;;} + measure: count {type:count drill_fields: [id, created_date, user_id, sale_price]} + measure: total_revenue {type:sum sql: ${sale_price} ;;} + measure: order_count {type:count_distinct sql: ${order_id} ;; drill_fields: [order_id, count]} +} diff --git a/examples/resources/probability_male.view.lkml b/examples/resources/probability_male.view.lkml new file mode 100644 index 0000000..33e625a --- /dev/null +++ b/examples/resources/probability_male.view.lkml @@ -0,0 +1,15 @@ +include: "/name_game2/name_game.explore" + +# https://master.dev.looker.com/explore/name_game2/name_game?qid=0Moo3wsmchrFsLofrNPPrS + +explore: probability_male {} +view: probability_male { + derived_table: { + explore_source: name_game { + column: name {} + column: male_percentage {} + } + } + dimension: name {hidden:yes} + dimension: male_percentage {type:number hidden: yes} +} diff --git a/examples/resources/products.view.lkml b/examples/resources/products.view.lkml new file mode 100644 index 0000000..b81fb14 --- /dev/null +++ b/examples/resources/products.view.lkml @@ -0,0 +1,18 @@ +view: products { + sql_table_name: thelook_web_analytics.products ;; + + dimension: id {primary_key:yes type:number sql: ${TABLE}.id ;;} + dimension: brand {sql: ${TABLE}.brand ;;} + dimension: category {sql: ${TABLE}.category ;;} + dimension: cost {type:number sql: ${TABLE}.cost ;;} + dimension: department {sql: ${TABLE}.department ;;} + dimension: distribution_center_id {type:number sql: ${TABLE}.distribution_center_id ;;} + dimension: name {sql: ${TABLE}.name ;;} + dimension: retail_price {type:number sql: ${TABLE}.retail_price ;;} + dimension: sku {sql: ${TABLE}.sku ;;} + + measure: brand_list {type:list list_field:brand} + measure: category_list {type:list list_field:category} + measure: count {type:count + drill_fields: [id, name, brand,category, inventory_items.count, order_items.count ]} +} diff --git a/examples/resources/thelook_web_analytics.model.lkml b/examples/resources/thelook_web_analytics.model.lkml new file mode 100644 index 0000000..cbe85a7 --- /dev/null +++ b/examples/resources/thelook_web_analytics.model.lkml @@ -0,0 +1,17 @@ +connection: "bigquery_publicdata_standard_sql" + +# include all the views +include: "*.view" + +# include all the dashboards +include: "*.dashboard" + +include: "*.explore" + +explore: event_sessions { + extends: [user_joins] + join: users {relationship:many_to_one sql_on: ${users.id} = ${event_sessions.user_id} ;; } + join: product_id {from:id relationship:one_to_many + sql: LEFT JOIN UNNEST(${event_sessions.product_ids_visited}) as product_id ;;} + join: products {relationship:many_to_one sql_on: ${product_id.id} = ${products.id} ;;} +} diff --git a/examples/resources/user_day_stats.view.lkml b/examples/resources/user_day_stats.view.lkml new file mode 100644 index 0000000..f301e1d --- /dev/null +++ b/examples/resources/user_day_stats.view.lkml @@ -0,0 +1,46 @@ +include: "users.view" +include: "user_joins.explore" + +explore: user_day_stats { + extends: [user_joins] + view_name: events + from: user_day_events + join: orders {from:user_day_orders relationship:one_to_one + sql_on: ${orders.user_id} = ${events.user_id} + AND ${orders.created_date} = ${events.created_date};;} + join: users {relationship:many_to_one + sql_on: ${users.id} = COALESCE(${events.user_id}, ${orders.user_id}) ;;} +} + +include: "events.explore" +include: "order_items.explore" + +view: user_day_events { + derived_table: { + explore_source: events { + column: user_id {field:events.user_id} + column: events {field:events.count} + column: created_date {field:events.created_date} + filters: {field:events.user_id value:"NOT NULL"} + } + } + dimension: user_id {} + dimension: created_date {} + measure: event_count {type:sum sql: ${TABLE}.events ;;} +} + + +view: user_day_orders { + derived_table: { + explore_source: order_items { + column: user_id { field: order_items.user_id } + column: created_date { field: order_items.created_date } + column: revenue { field: order_items.total_revenue } + column: orders { field: order_items.order_count } + } + } + dimension: created_date {} + dimension: user_id {} + measure: total_revenue {type:sum sql: ${TABLE}.revenue ;;} + measure: order_count {type:sum sql: ${TABLE}.orders ;;} +} diff --git a/examples/resources/user_event_attribution.view.lkml b/examples/resources/user_event_attribution.view.lkml new file mode 100644 index 0000000..61e7094 --- /dev/null +++ b/examples/resources/user_event_attribution.view.lkml @@ -0,0 +1,35 @@ +include: "events.explore" + +explore: user_events1 {hidden: yes} +view: user_events1 { + derived_table: { + explore_source: events { + column: user_id { field: events.user_id} + column: created_time {field: events.created_time} + column: traffic_source {field: events.traffic_source} + derived_column: event_sequence { + sql: ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY created_time);; + } + } + } + dimension: user_id {} + dimension: traffic_source {} + dimension: event_sequence {type: number} +} + +explore: user_event_attribution {hidden: no} +view: user_event_attribution { + derived_table: { + persist_for: "2 hours" + explore_source: user_events1 { + column: user_id {field: user_events1.user_id} + column: traffic_source {field: user_events1.traffic_source} + filters: { + field: user_events1.event_sequence + value: "1" + } + } + } + dimension: user_id {hidden: yes} + dimension: traffic_source {} +} diff --git a/examples/resources/user_joins.explore.lkml b/examples/resources/user_joins.explore.lkml new file mode 100644 index 0000000..30b5ad8 --- /dev/null +++ b/examples/resources/user_joins.explore.lkml @@ -0,0 +1,16 @@ +# All the facts we've built around the users +explore: user_joins { + extension: required + join: user_order_facts {relationship: many_to_one + sql_on: ${user_order_facts.user_id} = ${users.id} ;;} + join: user_event_attribution {view_label:"Users" relationship:many_to_one + sql_on: ${user_event_attribution.user_id} = ${users.id} ;;} + join: lifetime_brand {view_label:"Users" relationship:one_to_many + sql: LEFT JOIN UNNEST(SPLIT(${user_order_facts.lifetime_brands},'|RECORD|')) as lifetime_brand ;;} + join: lifetime_product_category { view_label:"Users" relationship:one_to_many + sql: LEFT JOIN UNNEST(SPLIT(${user_order_facts.lifetime_product_categories},'|RECORD|')) as lifetime_product_category ;;} + join: zip_demographics {relationship:many_to_one + sql_on: ${users.zip} = ${zip_demographics.zip};;} + join: probability_male {relationship:many_to_one + sql_on: ${users.first_name} = UPPER(${probability_male.name}) ;;} +} diff --git a/examples/resources/user_order_facts.view.lkml b/examples/resources/user_order_facts.view.lkml new file mode 100644 index 0000000..a6cf7c4 --- /dev/null +++ b/examples/resources/user_order_facts.view.lkml @@ -0,0 +1,18 @@ +include: "order_items.explore" + +view: user_order_facts { + derived_table: { + explore_source: order_items { + column: user_id {field:order_items.user_id} + column: lifetime_revenue {field:order_items.total_revenue} + column: lifetime_number_of_orders {field:order_items.order_count} + column: lifetime_product_categories {field:products.category_list} + column: lifetime_brands {field:products.brand_list} + } + } + dimension: user_id {hidden:yes} + dimension: lifetime_revenue {type:number} + dimension: lifetime_number_of_orders {type:number} + dimension: lifetime_product_categories {} + dimension: lifetime_brands {} +} diff --git a/examples/resources/user_order_sequence.view.lkml b/examples/resources/user_order_sequence.view.lkml new file mode 100644 index 0000000..2af4e38 --- /dev/null +++ b/examples/resources/user_order_sequence.view.lkml @@ -0,0 +1,16 @@ +include: "order_items.explore" + +explore: user_order_sequence {} +view: user_order_sequence { + derived_table: { + #persist_for: "2 hours" + explore_source: order_items { + column: user_id { field: order_items.user_id} + column: order_id {field: order_items.order_id} + column: created_time {field: order_items.created_time} + derived_column: user_sequence { sql: ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY created_time) ;;} + } + } + dimension: order_id {hidden:yes} + dimension: user_sequence {type:number} +} diff --git a/examples/resources/users.explore.lkml b/examples/resources/users.explore.lkml new file mode 100644 index 0000000..6ab4b29 --- /dev/null +++ b/examples/resources/users.explore.lkml @@ -0,0 +1,7 @@ +include: "user_joins.explore" +include: "users.view" + +explore: users { + view_name: users + extends: [user_joins] +} diff --git a/examples/resources/users.view.lkml b/examples/resources/users.view.lkml new file mode 100644 index 0000000..2f2b0e6 --- /dev/null +++ b/examples/resources/users.view.lkml @@ -0,0 +1,24 @@ +view: users { + sql_table_name: thelook_web_analytics.users ;; + + dimension: id {primary_key:yes} + dimension: age {type:number} + dimension: city {} + dimension: country {} + dimension_group: created {type:time sql:${TABLE}.created_at ;;} + dimension: email {} + dimension: first_name {} + dimension: gender {} + dimension: last_name {} + dimension: state {} + dimension: zip {type: zipcode} + + measure: average_home_value {type:average value_format_name: decimal_0 + sql: ${zip_demographics.median_home_value} ;;} + measure: probability_hispanic {type:average value_format_name: percent_2 + sql: ${zip_demographics.probability_hispanic} ;;} + measure: probability_male {type:average value_format_name: percent_2 + sql: ${probability_male.male_percentage} ;;} + measure: count {type:count + drill_fields: [id, last_name, first_name, events.count, order_items.count]} +} diff --git a/examples/resources/zip_demographics.view.lkml b/examples/resources/zip_demographics.view.lkml new file mode 100644 index 0000000..28b5d4a --- /dev/null +++ b/examples/resources/zip_demographics.view.lkml @@ -0,0 +1,33 @@ +include: "/acs/geo.explore" + +explore: zip_demographics {} +view: zip_demographics { + derived_table: { + persist_for: "24 hours" + explore_source: geo { + column: zip { field: tract_zcta_map.ZCTA5 } + column: median_home_value { field: median_value.weighted_median_value } + column: probability_hispanic { field: hispanic.percentage_selected } + column: total_population { field: population.total_base} + } + } + dimension: zip { + map_layer_name: us_zipcode_tabulation_areas + #hidden: yes + primary_key: yes + } + dimension: median_home_value { + type: number + #hidden: yes + } + dimension: probability_hispanic { + type: number + #hidden: yes + value_format_name: percent_2 + sql: ${TABLE}.probability_hispanic ;; + } + measure: total_population { + type: sum + sql: ${TABLE}.total_population ;; + } +} diff --git a/examples/test_lookml_migrator.py b/examples/test_lookml_migrator.py new file mode 100644 index 0000000..fe8a232 --- /dev/null +++ b/examples/test_lookml_migrator.py @@ -0,0 +1,25 @@ +import unittest +from unittest.mock import MagicMock, patch +import lkml + +# Mock data and function imports +from lookml_migrator import migrate_lookml + +def load_example_lookml(file_path): + with open(file_path, 'r') as f: + content = f.read() + return parse_lkml_content(content) + + +class TestMigrateLookML(unittest.TestCase): + def setUp(self): + # Mock the API client + self.mock_client = MagicMock() + + def test_migrate_lookml_model(self): + parsed_model = load_example_lookml("resources/thelook_web_analytics.model.lkml") + # Run migration + migrate_lookml(parsed_model, self.mock_client) + + # Check API calls + self.mock_client.create_model.assert_called_once_with(expected_omni_object)