diff --git a/jaffle_shop/models/final/finance/_exposures.yml b/jaffle_shop/models/final/finance/_exposures.yml new file mode 100644 index 000000000..b38da1da7 --- /dev/null +++ b/jaffle_shop/models/final/finance/_exposures.yml @@ -0,0 +1,12 @@ +version: 2 + +exposures: + - name: customer_return_total + description: Inksacio App to show the total sum of customer returns, per customer (for both completed and upcoming) + type: dashboard + url: https://inksacio.eks.octojaffle.engineering/customer_return_totals/ + owner: + name: 'Elliot Taylor' + email: elliot.taylor@octojaffle.com + depends_on: + - ref('fnl_finance_customerreturnstotal') \ No newline at end of file diff --git a/jaffle_shop/models/final/finance/_models.yml b/jaffle_shop/models/final/finance/_models.yml new file mode 100644 index 000000000..ced7510c5 --- /dev/null +++ b/jaffle_shop/models/final/finance/_models.yml @@ -0,0 +1,13 @@ +version: 2 + +models: + - name: fnl_finance_customerreturnstotal + description: > + This table gives one row per customer, with the total of their completed returns, + pending_returns and the sum of the 2 + columns: + - name: customer_id + description: primary key + tests: + - unique + - not_null diff --git a/jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql b/jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql new file mode 100644 index 000000000..a832f0dae --- /dev/null +++ b/jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql @@ -0,0 +1,14 @@ +{% set return_states = ['returned', 'return_pending'] %} + +select + orders.customer_id + + {% for return_state in return_states -%} + , SUM(orders.amount_dollars) FILTER (WHERE orders.status =' {{ return_state }}') AS {{ return_state }}_amount_dollars + {% endfor -%} + + , SUM(orders.amount_dollars) AS sum_return_amount_dollars + +from {{ ref('wh_orders') }} AS orders +where orders.status IN ('returned', 'return_pending') +GROUP BY orders.customer_id \ No newline at end of file diff --git a/jaffle_shop/models/final/sales/_exposures.yml b/jaffle_shop/models/final/sales/_exposures.yml new file mode 100644 index 000000000..c12bb1be5 --- /dev/null +++ b/jaffle_shop/models/final/sales/_exposures.yml @@ -0,0 +1,12 @@ +version: 2 + +exposures: + - name: new_customer_history + description: Inksacio App to show monthly new customer history + type: dashboard + url: https://inksacio.eks.octojaffle.engineering/new_customer_history/ + owner: + name: 'Elliot Taylor' + email: elliot.taylor@octojaffle.com + depends_on: + - ref('fnl_sales_newcustomerhistory') \ No newline at end of file diff --git a/jaffle_shop/models/final/sales/_models.yml b/jaffle_shop/models/final/sales/_models.yml new file mode 100644 index 000000000..8186417cf --- /dev/null +++ b/jaffle_shop/models/final/sales/_models.yml @@ -0,0 +1,13 @@ +version: 2 + +models: + - name: fnl_sales_newcustomerhistory + description: > + This table gives one row per truncated month with the count + of customers that have created their first order within this month + columns: + - name: first_order_month + description: primary key + tests: + - unique + - not_null \ No newline at end of file diff --git a/jaffle_shop/models/final/sales/fnl_sales_newcustomerhistory.sql b/jaffle_shop/models/final/sales/fnl_sales_newcustomerhistory.sql new file mode 100644 index 000000000..8258f25f4 --- /dev/null +++ b/jaffle_shop/models/final/sales/fnl_sales_newcustomerhistory.sql @@ -0,0 +1,4 @@ +select + date_trunc('month', first_order) AS first_order_month + , count(*) AS number_customers +from {{ ref('wh_customers') }} \ No newline at end of file diff --git a/jaffle_shop/models/staging/schema.yml b/jaffle_shop/models/staging/src_seed/_models.yml similarity index 68% rename from jaffle_shop/models/staging/schema.yml rename to jaffle_shop/models/staging/src_seed/_models.yml index c207e4cf5..981a7e3b9 100644 --- a/jaffle_shop/models/staging/schema.yml +++ b/jaffle_shop/models/staging/src_seed/_models.yml @@ -1,16 +1,11 @@ version: 2 models: - - name: stg_customers - columns: - - name: customer_id - tests: - - unique - - not_null - - name: stg_orders + description: staging layer for all orders columns: - name: order_id + description: Primary Key tests: - unique - not_null @@ -20,8 +15,10 @@ models: values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] - name: stg_payments + description: staging layer for all payments with a FK to order columns: - name: payment_id + description: Primary Key tests: - unique - not_null @@ -29,3 +26,12 @@ models: tests: - accepted_values: values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] + + - name: stg_customers + description: staging layer for all customers, PII hashed + columns: + - name: customer_id + description: Primary Key + tests: + - unique + - not_null \ No newline at end of file diff --git a/jaffle_shop/models/staging/src_seed/sensitive/_models.yml b/jaffle_shop/models/staging/src_seed/sensitive/_models.yml new file mode 100644 index 000000000..e75f52c6c --- /dev/null +++ b/jaffle_shop/models/staging/src_seed/sensitive/_models.yml @@ -0,0 +1,18 @@ +version: 2 + +models: + - name: stg_customers_pii + description: > + Table that includes all info about all customers, with PII hashed + columns: + - name: customer_id + description: Primary Key + tests: + - unique + - not_null + - name: first_name + meta: + sensitive: true + - name: last_name + meta: + sensitive: true \ No newline at end of file diff --git a/jaffle_shop/models/staging/stg_customers.sql b/jaffle_shop/models/staging/src_seed/sensitive/stg_customers_pii.sql similarity index 100% rename from jaffle_shop/models/staging/stg_customers.sql rename to jaffle_shop/models/staging/src_seed/sensitive/stg_customers_pii.sql diff --git a/jaffle_shop/models/staging/src_seed/stg_customers.sql b/jaffle_shop/models/staging/src_seed/stg_customers.sql new file mode 100644 index 000000000..b9eadce2c --- /dev/null +++ b/jaffle_shop/models/staging/src_seed/stg_customers.sql @@ -0,0 +1,3 @@ +SELECT + {{ hash_sensitive_columns('stg_customers_pii') }} +FROM {{ ref('stg_customers_pii') }} \ No newline at end of file diff --git a/jaffle_shop/models/staging/stg_orders.sql b/jaffle_shop/models/staging/src_seed/stg_orders.sql similarity index 100% rename from jaffle_shop/models/staging/stg_orders.sql rename to jaffle_shop/models/staging/src_seed/stg_orders.sql diff --git a/jaffle_shop/models/staging/stg_payments.sql b/jaffle_shop/models/staging/src_seed/stg_payments.sql similarity index 91% rename from jaffle_shop/models/staging/stg_payments.sql rename to jaffle_shop/models/staging/src_seed/stg_payments.sql index 700cf7f4f..502a3ecdc 100644 --- a/jaffle_shop/models/staging/stg_payments.sql +++ b/jaffle_shop/models/staging/src_seed/stg_payments.sql @@ -16,7 +16,7 @@ renamed as ( payment_method, -- `amount` is currently stored in cents, so we convert it to dollars - amount / 100 as amount + amount / 100 as amount_dollars from source diff --git a/jaffle_shop/models/docs.md b/jaffle_shop/models/warehouse/_docs.md similarity index 100% rename from jaffle_shop/models/docs.md rename to jaffle_shop/models/warehouse/_docs.md diff --git a/jaffle_shop/models/schema.yml b/jaffle_shop/models/warehouse/_models.yml similarity index 96% rename from jaffle_shop/models/schema.yml rename to jaffle_shop/models/warehouse/_models.yml index 381349cfd..d2708aa91 100644 --- a/jaffle_shop/models/schema.yml +++ b/jaffle_shop/models/warehouse/_models.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: customers + - name: wh_customers description: This table has basic information about a customer, as well as some derived facts based on a customer's orders columns: @@ -29,7 +29,7 @@ models: - name: total_order_amount description: Total value (AUD) of a customer's orders - - name: orders + - name: wh_orders description: This table has basic information about orders, as well as some derived facts based on payments columns: @@ -44,7 +44,7 @@ models: tests: - not_null - relationships: - to: ref('customers') + to: ref('wh_customers') field: customer_id - name: order_date diff --git a/jaffle_shop/models/customers.sql b/jaffle_shop/models/warehouse/wh_customers.sql similarity index 99% rename from jaffle_shop/models/customers.sql rename to jaffle_shop/models/warehouse/wh_customers.sql index 016a004fe..3f96d6e00 100644 --- a/jaffle_shop/models/customers.sql +++ b/jaffle_shop/models/warehouse/wh_customers.sql @@ -51,6 +51,7 @@ final as ( customers.customer_id, customers.first_name, customers.last_name, + customer_orders.first_order, customer_orders.most_recent_order, customer_orders.number_of_orders, diff --git a/jaffle_shop/models/orders.sql b/jaffle_shop/models/warehouse/wh_orders.sql similarity index 94% rename from jaffle_shop/models/orders.sql rename to jaffle_shop/models/warehouse/wh_orders.sql index cbb293491..b693d3775 100644 --- a/jaffle_shop/models/orders.sql +++ b/jaffle_shop/models/warehouse/wh_orders.sql @@ -43,7 +43,7 @@ final as ( {% endfor -%} - order_payments.total_amount as amount + order_payments.total_amount as amount_dollars from orders diff --git a/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv b/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv new file mode 100644 index 000000000..b9f94e579 --- /dev/null +++ b/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv @@ -0,0 +1,2 @@ +fct_name,column_name,id_to_exclude,comment +fct_staging_dependent_on_staging,parent,stg_customers_pii,Scrubbing pii permitted in staging layer. \ No newline at end of file