DATA_ETL modify

sync_amz_data/DataTransform/Data_ETL.py

@@ -78,13 +78,15 @@ class Common_ETLMethod(BaseClient):
     def get_keyOvalue(self,expressions, result: Literal['value', 'type']):
         if expressions not in [None,pd.NA,pd.NaT]:
             if len(expressions) > 1:
-                return [i[result] for i in expressions]
+                return [i.get(result) for i in expressions]
             if len(expressions) == 1:
-                sub_ = expressions[0][result]
-                if type(sub_) == str:
+                sub_ = expressions[0].get(result)
+                if sub_ is None:
+                    return None
+                elif type(sub_) == str:
                     return sub_
                 elif type(sub_) == list:
-                    return [i[result] for i in sub_]
+                    return [i.get(result) for i in sub_]
                 else:
                     return expressions
         else:
@@ -167,9 +169,14 @@ class SP_ETL(SPClient, Common_ETLMethod):
         return self.columnsName_modify(df_keywords_SP)
 
     def targets_ETL(self):
-        list_targets = list(self.iter_targets())
+        list_targets = list(self.iter_targets(**{"includeExtendedDataFields": True}))
         df_targets = pd.json_normalize(list_targets)
+        print("--")
         df_targets = self.TZ_Deal(df_targets, ["extendedData.creationDateTime", "extendedData.lastUpdateDateTime"])
+        df_targets['resolvedExpressions_type'] = df_targets['resolvedExpression'].map(
+            lambda x: self.get_keyOvalue(x, 'type'))
+        df_targets['resolvedExpressions_value'] = df_targets['resolvedExpression'].map(
+            lambda x: self.get_keyOvalue(x, 'value'))
         return self.columnsName_modify(df_targets)
 
     def budget_ETL(self, campaign_ids: list):
@@ -1002,8 +1009,8 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_targets = pd.json_normalize(list_targets)
         # df_targets = self.expression_split(df_targets, "resolvedExpression")
         df_targets = self.id_type_trans(df_targets)
-        df_targets['resolvedExpressions_type'] = df_targets['resolvedExpressions'].map(lambda x:self.get_keyOvalue(x,'type'))
-        df_targets['resolvedExpressions_value'] = df_targets['resolvedExpressions'].map(lambda x:self.get_keyOvalue(x,'value'))
+        df_targets['resolvedExpressions_type'] = df_targets['resolvedExpression'].map(lambda x:self.get_keyOvalue(x,'type'))
+        df_targets['resolvedExpressions_value'] = df_targets['resolvedExpression'].map(lambda x:self.get_keyOvalue(x,'value'))
         return self.columnsName_modify(df_targets)
 
     def budget_ETL(self, campaignsIds: list):
@@ -1784,9 +1791,12 @@ if __name__ == '__main__':
     conn = Common_ETLMethod(**AWS_CREDENTIALS).clickhouse_connect()
 
     # SD
-    ac_etl = SD_ETL(**AWS_CREDENTIALS)
-    ls = ac_etl.reportV2_campaignsRecord_t2_ETL(conn)
+    ac_etl = SB_ETL(**AWS_CREDENTIALS)
+    # ls = ac_etl.reportV2_campaignsRecord_t2_ETL(conn)
+    ls = ac_etl.targets_ETL()
     print(ls)
+    # ls.to_csv('temp.csv')
+    # print(ls)
     # print(ls.info())
     # print(ls.to_excel('obse11.xlsx'))
     # ac_etl.reportV2_campaignsRecord_t2_ETL(conn)