From b6772e53e50c5d05902388b08a00943f815728fa Mon Sep 17 00:00:00 2001
From: Elliot Taylor <elliot@taylor.org.uk>
Date: Wed, 4 Jan 2023 14:44:41 +0000
Subject: [PATCH 1/4] Refactor Repo to follow convention

---
 jaffle_shop/models/intermediate/_models.yml       |  0
 .../staging/{schema.yml => src_seed/_models.yml}  |  8 --------
 jaffle_shop/models/staging/src_seed/_sources.yml  |  0
 .../models/staging/src_seed/sensitive/_models.yml | 15 +++++++++++++++
 .../sensitive/stg_customers_pii.sql}              |  0
 .../models/staging/src_seed/stg_customers.sql     |  3 +++
 .../models/staging/{ => src_seed}/stg_orders.sql  |  0
 .../staging/{ => src_seed}/stg_payments.sql       |  0
 .../models/{schema.yml => warehouse/_models.yml}  |  0
 jaffle_shop/models/{ => warehouse}/customers.sql  |  0
 jaffle_shop/models/{ => warehouse}/docs.md        |  0
 jaffle_shop/models/{ => warehouse}/orders.sql     |  0
 .../seeds/dbt_project_evaluator_exceptions.csv    |  2 ++
 13 files changed, 20 insertions(+), 8 deletions(-)
 create mode 100644 jaffle_shop/models/intermediate/_models.yml
 rename jaffle_shop/models/staging/{schema.yml => src_seed/_models.yml} (81%)
 create mode 100644 jaffle_shop/models/staging/src_seed/_sources.yml
 create mode 100644 jaffle_shop/models/staging/src_seed/sensitive/_models.yml
 rename jaffle_shop/models/staging/{stg_customers.sql => src_seed/sensitive/stg_customers_pii.sql} (100%)
 create mode 100644 jaffle_shop/models/staging/src_seed/stg_customers.sql
 rename jaffle_shop/models/staging/{ => src_seed}/stg_orders.sql (100%)
 rename jaffle_shop/models/staging/{ => src_seed}/stg_payments.sql (100%)
 rename jaffle_shop/models/{schema.yml => warehouse/_models.yml} (100%)
 rename jaffle_shop/models/{ => warehouse}/customers.sql (100%)
 rename jaffle_shop/models/{ => warehouse}/docs.md (100%)
 rename jaffle_shop/models/{ => warehouse}/orders.sql (100%)
 create mode 100644 jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv

diff --git a/jaffle_shop/models/intermediate/_models.yml b/jaffle_shop/models/intermediate/_models.yml
new file mode 100644
index 000000000..e69de29bb
diff --git a/jaffle_shop/models/staging/schema.yml b/jaffle_shop/models/staging/src_seed/_models.yml
similarity index 81%
rename from jaffle_shop/models/staging/schema.yml
rename to jaffle_shop/models/staging/src_seed/_models.yml
index c207e4cf5..d55944bd1 100644
--- a/jaffle_shop/models/staging/schema.yml
+++ b/jaffle_shop/models/staging/src_seed/_models.yml
@@ -1,13 +1,5 @@
 version: 2
 
-models:
-  - name: stg_customers
-    columns:
-      - name: customer_id
-        tests:
-          - unique
-          - not_null
-
   - name: stg_orders
     columns:
       - name: order_id
diff --git a/jaffle_shop/models/staging/src_seed/_sources.yml b/jaffle_shop/models/staging/src_seed/_sources.yml
new file mode 100644
index 000000000..e69de29bb
diff --git a/jaffle_shop/models/staging/src_seed/sensitive/_models.yml b/jaffle_shop/models/staging/src_seed/sensitive/_models.yml
new file mode 100644
index 000000000..f5e4e844d
--- /dev/null
+++ b/jaffle_shop/models/staging/src_seed/sensitive/_models.yml
@@ -0,0 +1,15 @@
+version: 2
+
+models:
+  - name: stg_customers
+    columns:
+      - name: customer_id
+        tests:
+          - unique
+          - not_null
+      - name: first_name
+        meta:
+          sensitive: true
+      - name: last_name
+        meta:
+          sensitive: true
\ No newline at end of file
diff --git a/jaffle_shop/models/staging/stg_customers.sql b/jaffle_shop/models/staging/src_seed/sensitive/stg_customers_pii.sql
similarity index 100%
rename from jaffle_shop/models/staging/stg_customers.sql
rename to jaffle_shop/models/staging/src_seed/sensitive/stg_customers_pii.sql
diff --git a/jaffle_shop/models/staging/src_seed/stg_customers.sql b/jaffle_shop/models/staging/src_seed/stg_customers.sql
new file mode 100644
index 000000000..b9eadce2c
--- /dev/null
+++ b/jaffle_shop/models/staging/src_seed/stg_customers.sql
@@ -0,0 +1,3 @@
+SELECT
+    {{ hash_sensitive_columns('stg_customers_pii') }}
+FROM {{ ref('stg_customers_pii') }}
\ No newline at end of file
diff --git a/jaffle_shop/models/staging/stg_orders.sql b/jaffle_shop/models/staging/src_seed/stg_orders.sql
similarity index 100%
rename from jaffle_shop/models/staging/stg_orders.sql
rename to jaffle_shop/models/staging/src_seed/stg_orders.sql
diff --git a/jaffle_shop/models/staging/stg_payments.sql b/jaffle_shop/models/staging/src_seed/stg_payments.sql
similarity index 100%
rename from jaffle_shop/models/staging/stg_payments.sql
rename to jaffle_shop/models/staging/src_seed/stg_payments.sql
diff --git a/jaffle_shop/models/schema.yml b/jaffle_shop/models/warehouse/_models.yml
similarity index 100%
rename from jaffle_shop/models/schema.yml
rename to jaffle_shop/models/warehouse/_models.yml
diff --git a/jaffle_shop/models/customers.sql b/jaffle_shop/models/warehouse/customers.sql
similarity index 100%
rename from jaffle_shop/models/customers.sql
rename to jaffle_shop/models/warehouse/customers.sql
diff --git a/jaffle_shop/models/docs.md b/jaffle_shop/models/warehouse/docs.md
similarity index 100%
rename from jaffle_shop/models/docs.md
rename to jaffle_shop/models/warehouse/docs.md
diff --git a/jaffle_shop/models/orders.sql b/jaffle_shop/models/warehouse/orders.sql
similarity index 100%
rename from jaffle_shop/models/orders.sql
rename to jaffle_shop/models/warehouse/orders.sql
diff --git a/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv b/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv
new file mode 100644
index 000000000..b9f94e579
--- /dev/null
+++ b/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv
@@ -0,0 +1,2 @@
+fct_name,column_name,id_to_exclude,comment
+fct_staging_dependent_on_staging,parent,stg_customers_pii,Scrubbing pii permitted in staging layer.
\ No newline at end of file

From 866c78cc9b3a185ba822075bb31e5f145f5846a2 Mon Sep 17 00:00:00 2001
From: Elliot Taylor <elliot@taylor.org.uk>
Date: Wed, 4 Jan 2023 15:37:06 +0000
Subject: [PATCH 2/4] Add new models and add _models.yml files and remove
 _sources files

fix _models file
---
 jaffle_shop/models/final/finance/_models.yml   | 13 +++++++++++++
 .../fnl_finance_customerreturnstotal.sql       | 18 ++++++++++++++++++
 jaffle_shop/models/final/sales/_models.yml     | 13 +++++++++++++
 .../sales/fnl_sales_newcustomerhistory.sql     |  4 ++++
 jaffle_shop/models/intermediate/_models.yml    |  0
 .../models/staging/src_seed/_models.yml        |  1 +
 .../models/staging/src_seed/_sources.yml       |  0
 .../models/staging/src_seed/stg_payments.sql   |  2 +-
 .../models/warehouse/{docs.md => _docs.md}     |  0
 .../{customers.sql => wh_customers.sql}        |  1 +
 .../warehouse/{orders.sql => wh_orders.sql}    |  0
 11 files changed, 51 insertions(+), 1 deletion(-)
 create mode 100644 jaffle_shop/models/final/finance/_models.yml
 create mode 100644 jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql
 create mode 100644 jaffle_shop/models/final/sales/_models.yml
 create mode 100644 jaffle_shop/models/final/sales/fnl_sales_newcustomerhistory.sql
 delete mode 100644 jaffle_shop/models/intermediate/_models.yml
 delete mode 100644 jaffle_shop/models/staging/src_seed/_sources.yml
 rename jaffle_shop/models/warehouse/{docs.md => _docs.md} (100%)
 rename jaffle_shop/models/warehouse/{customers.sql => wh_customers.sql} (99%)
 rename jaffle_shop/models/warehouse/{orders.sql => wh_orders.sql} (100%)

diff --git a/jaffle_shop/models/final/finance/_models.yml b/jaffle_shop/models/final/finance/_models.yml
new file mode 100644
index 000000000..ced7510c5
--- /dev/null
+++ b/jaffle_shop/models/final/finance/_models.yml
@@ -0,0 +1,13 @@
+version: 2
+
+models:
+  - name: fnl_finance_customerreturnstotal
+    description: > 
+      This table gives one row per customer, with the total of their completed returns,
+      pending_returns and the sum of the 2
+    columns:
+      - name: customer_id
+        description: primary key
+        tests:
+          - unique
+          - not_null
diff --git a/jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql b/jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql
new file mode 100644
index 000000000..a3d2c59dc
--- /dev/null
+++ b/jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql
@@ -0,0 +1,18 @@
+{% set return_states = ['returned', 'return_pending'] %}
+
+select 
+    custs.customer_id 
+
+    {% for return_state in return_states -%}
+    , SUM(payments.amount_dollars) FILTER (WHERE orders.status =' {{ return_state }}') AS {{ return_state }}_amount_dollars
+    {% endfor -%}
+    
+    , SUM(payments.amount_dollars) AS sum_return_amount_dollars
+
+from {{ ref('wh_orders') }} AS orders
+    left join {{ ref('wh_customers') }} AS custs
+        on custs.customer_id = orders.customer_id
+    left join {{ ref('stg_payments') }} AS payments
+        on payments.order_id = orders.order_id
+where orders.status IN ('returned', 'return_pending')
+GROUP BY customer_id
\ No newline at end of file
diff --git a/jaffle_shop/models/final/sales/_models.yml b/jaffle_shop/models/final/sales/_models.yml
new file mode 100644
index 000000000..8186417cf
--- /dev/null
+++ b/jaffle_shop/models/final/sales/_models.yml
@@ -0,0 +1,13 @@
+version: 2
+
+models:
+  - name: fnl_sales_newcustomerhistory
+    description: > 
+      This table gives one row per truncated month with the count 
+      of customers that have created their first order within this month
+    columns:
+      - name: first_order_month
+        description: primary key
+        tests:
+          - unique
+          - not_null
\ No newline at end of file
diff --git a/jaffle_shop/models/final/sales/fnl_sales_newcustomerhistory.sql b/jaffle_shop/models/final/sales/fnl_sales_newcustomerhistory.sql
new file mode 100644
index 000000000..8258f25f4
--- /dev/null
+++ b/jaffle_shop/models/final/sales/fnl_sales_newcustomerhistory.sql
@@ -0,0 +1,4 @@
+select 
+    date_trunc('month', first_order) AS first_order_month
+    , count(*) AS number_customers
+from {{ ref('wh_customers') }}
\ No newline at end of file
diff --git a/jaffle_shop/models/intermediate/_models.yml b/jaffle_shop/models/intermediate/_models.yml
deleted file mode 100644
index e69de29bb..000000000
diff --git a/jaffle_shop/models/staging/src_seed/_models.yml b/jaffle_shop/models/staging/src_seed/_models.yml
index d55944bd1..46abf1529 100644
--- a/jaffle_shop/models/staging/src_seed/_models.yml
+++ b/jaffle_shop/models/staging/src_seed/_models.yml
@@ -1,5 +1,6 @@
 version: 2
 
+models:
   - name: stg_orders
     columns:
       - name: order_id
diff --git a/jaffle_shop/models/staging/src_seed/_sources.yml b/jaffle_shop/models/staging/src_seed/_sources.yml
deleted file mode 100644
index e69de29bb..000000000
diff --git a/jaffle_shop/models/staging/src_seed/stg_payments.sql b/jaffle_shop/models/staging/src_seed/stg_payments.sql
index 700cf7f4f..502a3ecdc 100644
--- a/jaffle_shop/models/staging/src_seed/stg_payments.sql
+++ b/jaffle_shop/models/staging/src_seed/stg_payments.sql
@@ -16,7 +16,7 @@ renamed as (
         payment_method,
 
         -- `amount` is currently stored in cents, so we convert it to dollars
-        amount / 100 as amount
+        amount / 100 as amount_dollars
 
     from source
 
diff --git a/jaffle_shop/models/warehouse/docs.md b/jaffle_shop/models/warehouse/_docs.md
similarity index 100%
rename from jaffle_shop/models/warehouse/docs.md
rename to jaffle_shop/models/warehouse/_docs.md
diff --git a/jaffle_shop/models/warehouse/customers.sql b/jaffle_shop/models/warehouse/wh_customers.sql
similarity index 99%
rename from jaffle_shop/models/warehouse/customers.sql
rename to jaffle_shop/models/warehouse/wh_customers.sql
index 016a004fe..3f96d6e00 100644
--- a/jaffle_shop/models/warehouse/customers.sql
+++ b/jaffle_shop/models/warehouse/wh_customers.sql
@@ -51,6 +51,7 @@ final as (
         customers.customer_id,
         customers.first_name,
         customers.last_name,
+        
         customer_orders.first_order,
         customer_orders.most_recent_order,
         customer_orders.number_of_orders,
diff --git a/jaffle_shop/models/warehouse/orders.sql b/jaffle_shop/models/warehouse/wh_orders.sql
similarity index 100%
rename from jaffle_shop/models/warehouse/orders.sql
rename to jaffle_shop/models/warehouse/wh_orders.sql

From 74d472bfbd13576079d66d5d5f54578c471ad2bb Mon Sep 17 00:00:00 2001
From: Elliot Taylor <elliot@taylor.org.uk>
Date: Wed, 4 Jan 2023 15:50:12 +0000
Subject: [PATCH 3/4] Add exposures

---
 jaffle_shop/models/final/finance/_exposures.yml      | 12 ++++++++++++
 jaffle_shop/models/final/sales/_exposures.yml        | 12 ++++++++++++
 .../models/staging/src_seed/sensitive/_models.yml    |  2 +-
 jaffle_shop/models/warehouse/_models.yml             |  6 +++---
 4 files changed, 28 insertions(+), 4 deletions(-)
 create mode 100644 jaffle_shop/models/final/finance/_exposures.yml
 create mode 100644 jaffle_shop/models/final/sales/_exposures.yml

diff --git a/jaffle_shop/models/final/finance/_exposures.yml b/jaffle_shop/models/final/finance/_exposures.yml
new file mode 100644
index 000000000..b38da1da7
--- /dev/null
+++ b/jaffle_shop/models/final/finance/_exposures.yml
@@ -0,0 +1,12 @@
+version: 2
+
+exposures:
+  - name: customer_return_total
+    description: Inksacio App to show the total sum of customer returns, per customer (for both completed and upcoming)
+    type: dashboard
+    url: https://inksacio.eks.octojaffle.engineering/customer_return_totals/
+    owner: 
+      name: 'Elliot Taylor'
+      email: elliot.taylor@octojaffle.com
+    depends_on:
+      - ref('fnl_finance_customerreturnstotal')
\ No newline at end of file
diff --git a/jaffle_shop/models/final/sales/_exposures.yml b/jaffle_shop/models/final/sales/_exposures.yml
new file mode 100644
index 000000000..c12bb1be5
--- /dev/null
+++ b/jaffle_shop/models/final/sales/_exposures.yml
@@ -0,0 +1,12 @@
+version: 2
+
+exposures:
+  - name: new_customer_history
+    description: Inksacio App to show monthly new customer history
+    type: dashboard
+    url: https://inksacio.eks.octojaffle.engineering/new_customer_history/
+    owner: 
+      name: 'Elliot Taylor'
+      email: elliot.taylor@octojaffle.com
+    depends_on:
+      - ref('fnl_sales_newcustomerhistory')
\ No newline at end of file
diff --git a/jaffle_shop/models/staging/src_seed/sensitive/_models.yml b/jaffle_shop/models/staging/src_seed/sensitive/_models.yml
index f5e4e844d..6649bf8af 100644
--- a/jaffle_shop/models/staging/src_seed/sensitive/_models.yml
+++ b/jaffle_shop/models/staging/src_seed/sensitive/_models.yml
@@ -1,7 +1,7 @@
 version: 2
 
 models:
-  - name: stg_customers
+  - name: stg_customers_pii
     columns:
       - name: customer_id
         tests:
diff --git a/jaffle_shop/models/warehouse/_models.yml b/jaffle_shop/models/warehouse/_models.yml
index 381349cfd..d2708aa91 100644
--- a/jaffle_shop/models/warehouse/_models.yml
+++ b/jaffle_shop/models/warehouse/_models.yml
@@ -1,7 +1,7 @@
 version: 2
 
 models:
-  - name: customers
+  - name: wh_customers
     description: This table has basic information about a customer, as well as some derived facts based on a customer's orders
 
     columns:
@@ -29,7 +29,7 @@ models:
       - name: total_order_amount
         description: Total value (AUD) of a customer's orders
 
-  - name: orders
+  - name: wh_orders
     description: This table has basic information about orders, as well as some derived facts based on payments
 
     columns:
@@ -44,7 +44,7 @@ models:
         tests:
           - not_null
           - relationships:
-              to: ref('customers')
+              to: ref('wh_customers')
               field: customer_id
 
       - name: order_date

From 3511e9c48baacb14aee7d7a6abac9c5d7ca101b1 Mon Sep 17 00:00:00 2001
From: Elliot Taylor <elliot@taylor.org.uk>
Date: Wed, 4 Jan 2023 16:11:25 +0000
Subject: [PATCH 4/4] Lineage Fixes

---
 .../finance/fnl_finance_customerreturnstotal.sql    | 12 ++++--------
 jaffle_shop/models/staging/src_seed/_models.yml     | 13 +++++++++++++
 .../models/staging/src_seed/sensitive/_models.yml   |  3 +++
 jaffle_shop/models/warehouse/wh_orders.sql          |  2 +-
 4 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql b/jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql
index a3d2c59dc..a832f0dae 100644
--- a/jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql
+++ b/jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql
@@ -1,18 +1,14 @@
 {% set return_states = ['returned', 'return_pending'] %}
 
 select 
-    custs.customer_id 
+    orders.customer_id 
 
     {% for return_state in return_states -%}
-    , SUM(payments.amount_dollars) FILTER (WHERE orders.status =' {{ return_state }}') AS {{ return_state }}_amount_dollars
+    , SUM(orders.amount_dollars) FILTER (WHERE orders.status =' {{ return_state }}') AS {{ return_state }}_amount_dollars
     {% endfor -%}
     
-    , SUM(payments.amount_dollars) AS sum_return_amount_dollars
+    , SUM(orders.amount_dollars) AS sum_return_amount_dollars
 
 from {{ ref('wh_orders') }} AS orders
-    left join {{ ref('wh_customers') }} AS custs
-        on custs.customer_id = orders.customer_id
-    left join {{ ref('stg_payments') }} AS payments
-        on payments.order_id = orders.order_id
 where orders.status IN ('returned', 'return_pending')
-GROUP BY customer_id
\ No newline at end of file
+GROUP BY orders.customer_id
\ No newline at end of file
diff --git a/jaffle_shop/models/staging/src_seed/_models.yml b/jaffle_shop/models/staging/src_seed/_models.yml
index 46abf1529..981a7e3b9 100644
--- a/jaffle_shop/models/staging/src_seed/_models.yml
+++ b/jaffle_shop/models/staging/src_seed/_models.yml
@@ -2,8 +2,10 @@ version: 2
 
 models:
   - name: stg_orders
+    description: staging layer for all orders
     columns:
       - name: order_id
+        description: Primary Key
         tests:
           - unique
           - not_null
@@ -13,8 +15,10 @@ models:
               values: ['placed', 'shipped', 'completed', 'return_pending', 'returned']
 
   - name: stg_payments
+    description: staging layer for all payments with a FK to order
     columns:
       - name: payment_id
+        description: Primary Key
         tests:
           - unique
           - not_null
@@ -22,3 +26,12 @@ models:
         tests:
           - accepted_values:
               values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card']
+
+  - name: stg_customers
+    description: staging layer for all customers, PII hashed
+    columns:
+      - name: customer_id
+        description: Primary Key
+        tests:
+          - unique
+          - not_null
\ No newline at end of file
diff --git a/jaffle_shop/models/staging/src_seed/sensitive/_models.yml b/jaffle_shop/models/staging/src_seed/sensitive/_models.yml
index 6649bf8af..e75f52c6c 100644
--- a/jaffle_shop/models/staging/src_seed/sensitive/_models.yml
+++ b/jaffle_shop/models/staging/src_seed/sensitive/_models.yml
@@ -2,8 +2,11 @@ version: 2
 
 models:
   - name: stg_customers_pii
+    description: > 
+      Table that includes all info about all customers, with PII hashed
     columns:
       - name: customer_id
+        description: Primary Key
         tests:
           - unique
           - not_null
diff --git a/jaffle_shop/models/warehouse/wh_orders.sql b/jaffle_shop/models/warehouse/wh_orders.sql
index cbb293491..b693d3775 100644
--- a/jaffle_shop/models/warehouse/wh_orders.sql
+++ b/jaffle_shop/models/warehouse/wh_orders.sql
@@ -43,7 +43,7 @@ final as (
 
         {% endfor -%}
 
-        order_payments.total_amount as amount
+        order_payments.total_amount as amount_dollars
 
     from orders