|
@@ -75,7 +75,8 @@ class Common_ETLMethod(BaseClient):
|
|
|
df.drop(columns=[segment], inplace=True)
|
|
|
return df
|
|
|
|
|
|
- def get_keyOvalue(self,expressions, result: Literal['value', 'type']):
|
|
|
+ @classmethod
|
|
|
+ def get_keyOvalue(cls,expressions, result: Literal['value', 'type']):
|
|
|
if expressions not in [None,pd.NA,pd.NaT]:
|
|
|
if len(expressions) > 1:
|
|
|
return [i.get(result) for i in expressions]
|
|
@@ -209,13 +210,13 @@ class SP_ETL(SPClient, Common_ETLMethod):
|
|
|
df_targets = pd.json_normalize(list_targets)
|
|
|
df_targets = self.TZ_Deal(df_targets, ["extendedData.creationDateTime", "extendedData.lastUpdateDateTime"])
|
|
|
df_targets['resolvedExpressions_type'] = df_targets['resolvedExpression'].map(
|
|
|
- lambda x: self.get_keyOvalue(x, 'type'))
|
|
|
+ lambda x: Common_ETLMethod.get_keyOvalue(x, 'type'))
|
|
|
df_targets['resolvedExpressions_value'] = df_targets['resolvedExpression'].map(
|
|
|
- lambda x: self.get_keyOvalue(x, 'value'))
|
|
|
+ lambda x: Common_ETLMethod.get_keyOvalue(x, 'value'))
|
|
|
df_targets['expression_type'] = df_targets['expression'].map(
|
|
|
- lambda x: self.get_keyOvalue(x, 'type'))
|
|
|
+ lambda x: Common_ETLMethod.get_keyOvalue(x, 'type'))
|
|
|
df_targets['expression_value'] = df_targets['expression'].map(
|
|
|
- lambda x: self.get_keyOvalue(x, 'value'))
|
|
|
+ lambda x: Common_ETLMethod.get_keyOvalue(x, 'value'))
|
|
|
return self.columnsName_modify(df_targets)
|
|
|
|
|
|
def negative_targets_ETL(self):
|
|
@@ -223,13 +224,13 @@ class SP_ETL(SPClient, Common_ETLMethod):
|
|
|
df_targets = pd.json_normalize(list_targets)
|
|
|
df_targets = self.TZ_Deal(df_targets, ["extendedData.creationDateTime", "extendedData.lastUpdateDateTime"])
|
|
|
df_targets['resolvedExpressions_type'] = df_targets['resolvedExpression'].map(
|
|
|
- lambda x: self.get_keyOvalue(x, 'type'))
|
|
|
+ lambda x: Common_ETLMethod.get_keyOvalue(x, 'type'))
|
|
|
df_targets['resolvedExpressions_value'] = df_targets['resolvedExpression'].map(
|
|
|
- lambda x: self.get_keyOvalue(x, 'value'))
|
|
|
+ lambda x: Common_ETLMethod.get_keyOvalue(x, 'value'))
|
|
|
df_targets['expression_type'] = df_targets['expression'].map(
|
|
|
- lambda x: self.get_keyOvalue(x, 'type'))
|
|
|
+ lambda x: Common_ETLMethod.get_keyOvalue(x, 'type'))
|
|
|
df_targets['expression_value'] = df_targets['expression'].map(
|
|
|
- lambda x: self.get_keyOvalue(x, 'value'))
|
|
|
+ lambda x: Common_ETLMethod.get_keyOvalue(x, 'value'))
|
|
|
return self.columnsName_modify(df_targets)
|
|
|
|
|
|
def budget_ETL(self, campaign_ids: list):
|
|
@@ -531,8 +532,8 @@ class SB_ETL(SBClient, Common_ETLMethod):
|
|
|
# df_targets = self.TZ_Deal(df_targets, ["extendedData.creationDateTime", "extendedData.lastUpdateDateTime"])
|
|
|
# df_targets = self.expression_split(df_targets, "resolvedExpressions")
|
|
|
df_targets = self.id_type_trans(df_targets)
|
|
|
- df_targets['resolvedExpressions_type'] = df_targets['resolvedExpressions'].map(lambda x:self.get_keyOvalue(x,'type'))
|
|
|
- df_targets['resolvedExpressions_value'] = df_targets['resolvedExpressions'].map(lambda x:self.get_keyOvalue(x,'value'))
|
|
|
+ df_targets['resolvedExpressions_type'] = df_targets['resolvedExpressions'].map(lambda x:Common_ETLMethod.get_keyOvalue(x,'type'))
|
|
|
+ df_targets['resolvedExpressions_value'] = df_targets['resolvedExpressions'].map(lambda x:Common_ETLMethod.get_keyOvalue(x,'value'))
|
|
|
return self.columnsName_modify(df_targets)
|
|
|
|
|
|
def budget_ETL(self, campaign_ids: list):
|
|
@@ -1239,8 +1240,8 @@ class SD_ETL(SDClient, Common_ETLMethod):
|
|
|
df_targets = pd.json_normalize(list_targets)
|
|
|
# df_targets = self.expression_split(df_targets, "resolvedExpression")
|
|
|
df_targets = self.id_type_trans(df_targets)
|
|
|
- df_targets['resolvedExpressions_type'] = df_targets['resolvedExpression'].map(lambda x:self.get_keyOvalue(x,'type'))
|
|
|
- df_targets['resolvedExpressions_value'] = df_targets['resolvedExpression'].map(lambda x:self.get_keyOvalue(x,'value'))
|
|
|
+ df_targets['resolvedExpressions_type'] = df_targets['resolvedExpression'].map(lambda x:Common_ETLMethod.get_keyOvalue(x,'type'))
|
|
|
+ df_targets['resolvedExpressions_value'] = df_targets['resolvedExpression'].map(lambda x:Common_ETLMethod.get_keyOvalue(x,'value'))
|
|
|
return self.columnsName_modify(df_targets)
|
|
|
|
|
|
def budget_ETL(self, campaignsIds: list):
|
|
@@ -2026,15 +2027,15 @@ if __name__ == '__main__':
|
|
|
}
|
|
|
conn = Common_ETLMethod(**AWS_CREDENTIALS).clickhouse_connect()
|
|
|
|
|
|
- sb_ = SB_ETL(**AWS_CREDENTIALS)
|
|
|
-
|
|
|
+ sb_ = SP_ETL(**AWS_CREDENTIALS)
|
|
|
+ print(sb_.targets_ETL())
|
|
|
# list_date = ['2023-11-20']
|
|
|
- list_date = [f'2023-11-{"0" + str(i) if len(str(i)) == 1 else i}' for i in range(1, 22)]
|
|
|
- for date_ in list_date:
|
|
|
- print(date_)
|
|
|
- print(date_.replace("-",""))
|
|
|
- sb_ = SB_ETL(**AWS_CREDENTIALS)
|
|
|
- rel = sb_.reportV3_sbAds_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
|
|
|
- print(rel)
|
|
|
- print(rel.info())
|
|
|
- print(rel.columns)
|
|
|
+ # list_date = [f'2023-11-{"0" + str(i) if len(str(i)) == 1 else i}' for i in range(1, 22)]
|
|
|
+ # for date_ in list_date:
|
|
|
+ # print(date_)
|
|
|
+ # print(date_.replace("-",""))
|
|
|
+ # sb_ = SB_ETL(**AWS_CREDENTIALS)
|
|
|
+ # rel = sb_.reportV3_sbAds_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
|
|
|
+ # print(rel)
|
|
|
+ # print(rel.info())
|
|
|
+ # print(rel.columns)
|