exploreomni · jdavidson · Jan 6, 2025
diff --git a/examples/lookml_migrator.py b/examples/lookml_migrator.py
@@ -0,0 +1,201 @@
+import requests
+import os
+import base64
+import lkml
+import sys
+import argparse
+import pdb
+from omni_python_sdk import OmniAPI
+
+
+# Fetch the token if available
+GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
+
+# Set up headers with conditional authentication
+headers = {
+    "Accept": "application/vnd.github.v3+json"
+}
+if GITHUB_TOKEN:
+    headers["Authorization"] = f"token {GITHUB_TOKEN}"
+
+def base_url(repo: str):
+    return f"https://api.github.com/repos/{repo}"
+
+def find_lkml_files_in_repo(repo: str, branch: str = "main"):
+    lkml_files = []
+    url = f"{base_url(repo)}/git/trees/{branch}?recursive=1"  # Adjust 'main' if you’re using a different branch
+
+    response = requests.get(url, headers=headers)
+    response.raise_for_status()
+
+    # Search for .lkml files in the tree response
+    files = response.json().get("tree", [])
+    for file in files:
+        if file["path"].endswith(".lkml"):
+            lkml_files.append(file["path"])
+
+    return lkml_files
+
+def get_file_content(repo: str, file_path: str):
+    url = f"{base_url(repo)}/contents/{file_path}"
+    response = requests.get(url, headers=headers)
+    response.raise_for_status()
+
+    content = response.json().get("content")
+    if content:
+        # Decode the Base64 content
+        decoded_content = base64.b64decode(content).decode("utf-8")
+        return decoded_content
+    return None
+
+def parse_lkml_content(content):
+    parsed = lkml.load(content)
+    return parsed
+
+def parse_lookml(repo: str, branch: str):
+    lkml_files = find_lkml_files_in_repo(repo, branch)
+    parsed_objects = []
+
+    for file_path in lkml_files:
+        content = get_file_content(repo, file_path)
+        if content:
+            parsed = parse_lkml_content(content)
+            parsed_objects.append(parsed)
+
+    # Continue with mapping and creating objects as needed
+    return parsed_objects
+
+def map_lookml_model_to_omni(parsed_model):
+    # Map LookML model to Omni model
+    pass
+
+
+# explore_source: identifier {
+#   bind_all_filters: yes
+#   column: identifier {
+#     field: field_name
+#   }
+#   derived_column: identifier {
+#     sql: SQL expression ;;
+#   }
+#   expression_custom_filter: [custom filter expression]
+#   filters: [field_name_1: "string", field_name_2: "string", ...]
+#   limit: number
+#   sorts: [field_name_1: asc | desc, field_name_2: asc | desc, ...]
+#   timezone: "string"
+# }
+def map_lookml_query_to_omni(parsed_query):
+    omni_query = {
+        "topic": parsed_query["explore_source"]["name"]
+    }
+    omni_query["fields"]= {}
+    for column in parsed_query["columns"]:
+        if "field" in column:
+            omni_query["fields"][column["field"]] = column["name"]
+        else:
+            omni_query["fields"][column["name"]] = column["name"]
+    if "derived_columns" in parsed_query:
+        for derived_column in parsed_query["derived_columns"]:
+            # todo map derived column to omni
+            omni_query["fields"][derived_column["name"]] = derived_column["sql"]
+    if "filters_all" in parsed_query:
+        pass
+    if "persist_for" in parsed_query:
+        pass
+    if "bind_filters__all" in parsed_query:
+        pass
+    if "limit" in parsed_query:
+        pass
+    if "sorts" in parsed_query:
+        pass
+    if "timezone" in parsed_query:
+        pass
+
+
+def map_lookml_view_to_omni(parsed_view):
+    omni_view = {
+        "name": parsed_view["name"]
+    }
+
+    if "sql_table_name" in parsed_view:
+        # if schema is present
+        if parsed_view["sql_table_name"].count(".") > 0:
+            omni_view["schema"] = parsed_view["sql_table_name"].split(".")[0]
+            omni_view["table"] = parsed_view["sql_table_name"].split(".")[1]
+        else:
+            omni_view["table"] = parsed_view["sql_table_name"]
+    elif "derived_table" in parsed_view:
+        if "sql" in parsed_view["derived_table"]:
+            omni_view["sql"] = parsed_view["derived_table"]["sql"]
+        elif "explore_source" in parsed_view["derived_table"]:
+            omni_view["query"] = map_lookml_query_to_omni(parsed_view["derived_table"]["explore_source"])
+        else:
+            # Handle derived_table: sql_trigger
+            pass
+
+    # Map LookML view to Omni view
+    if "drill_fields" in parsed_view:
+        omni_view["drill_fields"] = parsed_view["drill_fields"]
+    if "label" in parsed_view:
+        omni_view["label"] = parsed_view["label"]
+    if "required_access_grants" in parsed_view:
+        # TODO map required access grants
+        # omni_view["required_access_grants"] = parsed_view["required_access_grants"]
+        pass
+
+    # Map fields
+    if "dimensions" in parsed_view:
+        omni_view["dimensions"] = [map_lookml_dimension_to_omni(dim) for dim in parsed_view["dimensions"]]
+
+    if "dimension_groups" in parsed_view:
+        dim_groups = [map_lookml_dimension_group_to_omni(dim_group) for dim_group in parsed_view["dimension_groups"]]
+
+    if "dimensions" in omni_view:
+        omni_view["dimensions"] += dim_groups
+    else:
+        omni_view["dimensions"] = dim_groups
+
+    if "measures" in parsed_view:
+        omni_view["measures"] = [map_lookml_measure_to_omni(measure) for measure in parsed_view["measures"]]
+    return omni_view
+
+def map_lookml_dimension_to_omni(parsed_field):
+    # Map LookML dimension to Omni dimension
+    pass
+
+def map_lookml_measure_to_omni(parsed_field):
+    # Map LookML measure to Omni measure
+    pass
+
+def map_lookml_dimension_group_to_omni(parsed_field):
+    # Map LookML dimension group to Omni date field
+    pass
+
+def map_lookml_explore_to_omni(parsed_explore):
+    # Map LookML explore to Omni topic
+    pass
+
+def migrate_lookml(repo: str, branch: str, client: OmniAPI, connection_id: str):
+    parsed_objects = parse_lookml(repo, branch)
+    # migrate lookml views
+
+    views = [view for obj in parsed_objects if "views" in obj for view in obj["views"]]
+
+
+if __name__ == "__main__":
+
+  parser = argparse.ArgumentParser(description="LookML Migrator")
+
+  # Define named arguments with the -- prefix
+  parser.add_argument("--github_repo", type=str, help="LookML GitHub repository", required=True)
+  parser.add_argument("--branch", type=str, help="Branch name", default="main")
+  parser.add_argument("--omni_api_key", type=str, help="Omni API key", required=True)
+  parser.add_argument("--omni_base_url", type=str, help="Omni base URL", required=True)
+  parser.add_argument("--omni_connection_id", type=str, help="Omni Connection ID", required=True)
+
+  args = parser.parse_args()
+
+  # Initialize the OmniAPI client
+  client = OmniAPI(args.omni_api_key, args.omni_base_url)
+
+  migrate_lookml(args.github_repo, args.branch, client, args.omni_connection_id,)
diff --git a/examples/resources/daily_funnel.view.lkml b/examples/resources/daily_funnel.view.lkml
@@ -0,0 +1,56 @@
+include: "order_items.explore"
+include: "users.explore"
+
+explore: daily_funnel {
+  join: funnel_orders {relationship:one_to_one
+    sql_on: ${funnel_orders.order_date} = ${daily_funnel.date} ;;}
+  join: funnel_signups {relationship:one_to_one
+    sql_on: ${funnel_signups.signup_date} = ${daily_funnel.date} ;;}
+}
+
+
+# Start with a Date table
+view: daily_funnel {
+  derived_table: {
+    sql:
+     SELECT * FROM (
+       SELECT
+           DATE_ADD('2010-01-01', INTERVAL num DAY) as d
+       FROM UNNEST(GENERATE_ARRAY(1,10000)) num
+     )
+     WHERE {% condition daily_funnel.filter_date %}
+        TIMESTAMP(d)
+        {% endcondition %}
+    ;;
+  }
+  filter: filter_date {type:date datatype:date}
+  dimension: date {can_filter:no  sql: ${TABLE}.d ;;}
+}
+
+# Join orders by day
+view: funnel_orders {
+  derived_table: {
+    explore_source: order_items {
+      column: order_date {field:order_items.created_date}
+      column: order_count {field:order_items.order_count}
+      bind_filters: {from_field: daily_funnel.filter_date
+        to_field: order_items.created_date }
+    }
+  }
+  dimension: order_date {}
+  dimension: order_count {type: number}
+}
+
+# Join user signups.
+view: funnel_signups {
+  derived_table: {
+    explore_source: users {
+      column: signup_date {field:users.created_date}
+      column: signup_count {field:users.count}
+      bind_filters: {from_field:daily_funnel.filter_date
+        to_field:users.created_date}
+    }
+  }
+  dimension: signup_date {}
+  dimension: signup_count {type: number}
+}
diff --git a/examples/resources/distribution_centers.view.lkml b/examples/resources/distribution_centers.view.lkml
@@ -0,0 +1,9 @@
+view: distribution_centers {
+  sql_table_name: thelook_web_analytics.distribution_centers ;;
+
+  dimension: id {primary_key:yes  type:number}
+  dimension: latitude {type:number}
+  dimension: longitude {type:number}
+  dimension: name {}
+  measure: count {type:count  drill_fields:[id, name, products.count]}
+}
diff --git a/examples/resources/event_sessions.view.lkml b/examples/resources/event_sessions.view.lkml
@@ -0,0 +1,44 @@
+include: "products.view"
+include: "users.view"
+include: "events.explore"
+
+view: event_sessions {
+  derived_table: {
+    #persist_for: "2 hours"
+    explore_source: events {
+      column: session_id { field: events.session_id }
+      column: event_types { field: events.event_types }
+      column: session_time { field: events.minimum_time }
+      column: session_end_time {field: events.max_time}
+      column: ip_addresses {field: events.ip_addresses }
+      column: user_id {field: events.first_user_id }
+      column: product_ids_visited {field: events.product_ids_visited}
+      derived_column: session_sequence {
+        sql: ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY session_time) ;;
+      }
+    }
+  }
+  dimension: session_id {primary_key:yes}
+  dimension: event_types {}
+  dimension_group: session {type:time  sql: ${TABLE}.session_time ;;}
+  dimension: session_end_time {hidden:yes}
+  dimension: ip_addresses {}
+  dimension: user_id {}
+  dimension: product_ids_visited {}
+  dimension: session_sequence {type:number}
+  dimension: session_length {type:number
+    sql: TIMESTAMP_DIFF(${session_end_time},${session_raw}, SECOND) ;;}
+  dimension: session_length_tiered {type:tier  tiers: [0,60,120]  sql: ${session_length} ;;}
+
+  measure: count_sessions {type:count  drill_fields:[session*]}
+  measure: count_sessions_with_cart {type:count  drill_fields:[session*]
+    filters: {field:event_types  value:"%Cart%"}}
+  measure: count_sessions_with_purchases {type:count  drill_fields: [session*]
+    filters: {field:event_types value:"%Purchase%"}}
+
+  set: session{ fields:[session_time, session_id, user_id, event_types]}
+}
+
+view: id {
+  dimension: id {sql: ${TABLE} ;;}
+}
diff --git a/examples/resources/events.explore.lkml b/examples/resources/events.explore.lkml
@@ -0,0 +1,8 @@
+include: "users.view"
+include: "events.view"
+include: "user_joins.explore"
+
+explore: events {
+  extends: [user_joins]
+  join: users {relationship:many_to_one  sql_on: ${events.user_id} = ${users.id} ;;}
+}
diff --git a/examples/resources/events.view.lkml b/examples/resources/events.view.lkml
@@ -0,0 +1,44 @@
+view: events {
+  sql_table_name: thelook_web_analytics.events ;;
+
+  dimension: id {primary_key:yes  type:number}
+  dimension: browser {}
+  dimension: city {}
+  dimension: country {}
+  dimension_group: created {type:time  sql: ${TABLE}.created_at ;;}
+  dimension: event_type {}
+  dimension: ip_address {}
+  dimension: latitude {type:number}
+  dimension: longitude {type:number}
+  dimension: os {}
+  dimension: sequence_number {type:number}
+  dimension: session_id {}
+  dimension: state {}
+  dimension: traffic_source {}
+  dimension: uri {}
+  dimension: user_id {type:number  sql: CAST(REGEXP_EXTRACT(${TABLE}.user_id, r'\d+') AS INT64) ;;}
+  dimension: zip {}
+
+  measure: count {type:count  drill_fields:[id, users.last_name, users.id, users.first_name]}
+
+
+  #
+  #  Sessionization Fields.
+  #
+  measure: minimum_time {sql: MIN(${created_raw}) ;;}
+  measure: max_time {sql: MAX(${created_raw}) ;;}
+  measure: ip_addresses {sql: ARRAY_TO_STRING(ARRAY_AGG(DISTINCT ${ip_address}),'|') ;;}
+  measure: event_types {sql: ARRAY_TO_STRING(ARRAY_AGG(DISTINCT ${event_type}),'|') ;;}
+
+  # code defensivly.  Should only have one user_id on a session, but just
+  # in case, we'll take the first one we see.
+  measure: first_user_id {type:min sql: ${user_id} ;;}
+
+  # parse the product_id out of the urls visited and return an array of them.
+  measure: product_ids_visited {
+    sql: ARRAY_AGG(DISTINCT
+            CAST(REGEXP_EXTRACT(${uri}, r'/product/(\d+)') AS INT64)
+          IGNORE NULLS) ;;
+  }
+
+}
diff --git a/examples/resources/filtered_lookml_dt.view.lkml b/examples/resources/filtered_lookml_dt.view.lkml
@@ -0,0 +1,22 @@
+include: "users.explore.lkml"
+
+explore: filtered_lookml_dt {}
+
+view: filtered_lookml_dt {
+  derived_table: {
+    explore_source: users {
+      column: age {field: users.age}
+      column: people {field: users.count}
+      bind_filters: {
+        to_field: users.created_date
+        from_field: filtered_lookml_dt.filter_date
+      }
+    }
+  }
+
+  filter: filter_date {
+    type: date
+  }
+  dimension: age {type: number}
+  dimension: people {type: number}
+}