Hi I am trying to get a DF and take the las date of the past DF and make a query of the form QUERY = ('SELECT * FROM properati-dw.public.ads WHERE place.l1 = "Colombia" AND start_date > DATE('+str(old_date)+");")
and then merging thew old df and the new DataFrame
past_df = pd.read_pickle(
"/Users/jagudelo/Library/CloudStorage/OneDrive-UniversidaddelosAndes/Observatorio CEDE_NEW/Propiedades/Properati/To_CTM/Properaty.pkl")
fecha=past_df["start_date"].max()
client = bigquery.Client(credentials = credentials,project=project_id)
QUERY = ('SELECT * FROM `properati-dw.public.ads` WHERE place.l1 = "Colombia" AND start_date > DATE('+str(fecha)+");")
query_job = client.query(QUERY)
rows = query_job.result()
df=client.query(QUERY).to_dataframe()
dw=pd.concat([df.drop(["place"], axis=1), df['place'].apply(pd.Series)], axis=1)
dw=pd.concat([dw.drop(["property"], axis=1), dw['property'].apply(pd.Series)], axis=1)
head=dw.head(2)
new_df = dw.loc[:,~dw.columns.duplicated()]
data=pd.concat([past_df,new_df])
