%load_ext watermark

%watermark -v -p numpy,pandas,geopandas,matplotlib,json,requests,sodapy

# listing items in data folder
%ls data/

# read data as a dataframe
df = pd.read_csv('data/sample-data.csv')

# previewing first five rows in data
df.head()

rows, columns = df.shape
print('This dataset has {:,} rows and {:,} columns.'.format(rows, columns))

# read data as a dataframe
df = pd.read_json('data/sample-data.json')

# previewing first five rows in data
df.head()

rows, columns = df.shape
print('This dataset has {:,} rows and {:,} columns.'.format(rows, columns))

# read data as a dataframe
df = pd.read_excel('data/sample-data.xlsx')

# previewing first five rows in data
df.head()

rows, columns = df.shape
print('This dataset has {:,} rows and {:,} columns.'.format(rows, columns))

# read data as a geodataframe
path = 'data/sample-data.shp'
gdf = gpd.read_file(path)

# previewing first five rows in data
gdf.head()

rows, columns = gdf.shape
print('This dataset has {:,} rows and {:,} columns.'.format(rows, columns))

path = 'data/sample-data.geojson'
gdf = gpd.read_file(path)

gdf.head()

rows, columns = gdf.shape
print('This dataset has {:,} rows and {:,} columns.'.format(rows, columns))

# download url of dataset
url = 'https://www1.nyc.gov/assets/planning/download/zip/data-maps/open-data/nyc_pluto_20v1_csv.zip'

# a path to our downloads folder 
downloads_path = str(Path.home() / "Downloads")

# specify file name
filename = 'PLUTO.gz'

# a path to our file from our downloads path
filepath = os.path.join(downloads_path, filename)

# retrieving data 
urllib.request.urlretrieve(url, filepath)

# open zipfile and saving items in our zipfolder
items = zipfile.ZipFile(filepath)

# available files in the container
print(items.namelist())

# a path to our file from our downloads path
file = 'pluto_20v1.csv'
filepath = os.path.join(downloads_path, file)
pluto_data = pd.read_csv(filepath, low_memory=False)

pluto_data.head()

rows, columns = pluto_data.shape
print('This dataset has {:,} rows and {:,} columns.'.format(rows, columns))

# list files in this file path
%ls data/unzipped-data/

# read data as a dataframe
path = 'data/unzipped-data/sample-data.csv'
sample_buidlings = pd.read_csv(path)

# previewing first five rows of data
sample_buidlings.head()

rows, columns = sample_buidlings.shape
print('This dataset has {:,} rows and {:,} columns.'.format(rows, columns))

# our download link
url = 'https://www1.nyc.gov/assets/planning/download/zip/data-maps/open-data/nyc_pluto_20v1_csv.zip'

# reading in our zipfile data in-memory
content = requests.get(url)
zf = ZipFile(BytesIO(content.content))

# printing files in our zipfile
for item in zf.namelist():
    print("File in zip: {}".format(item))

# read our csv data into a dataframe from our zipfile
file = 'pluto_20v1.csv'
pluto_data = pd.read_csv(zf.open(file), low_memory=False)

# previewing the first five rows of data
pluto_data.head()

rows, columns = pluto_data.shape
print('This dataset has {:,} rows and {:,} columns.'.format(rows, columns))

# reading in data as a url from NYC Open Data
url = 'https://data.cityofnewyork.us/api/views/qb5r-6dgf/rows.csv?accessType=DOWNLOAD'

# saving data as a pandas dataframe named 'df_csv'
df_csv = pd.read_csv(url)

# previewing the first five rows 
df_csv.head()

rows, columns = df_csv.shape
print('This dataset has {:,} rows and {:,} columns.'.format(rows, columns))

url = 'https://data.cityofnewyork.us/api/views/qb5r-6dgf/rows.json?accessType=DOWNLOAD'

# loads a json object as a python object
with urllib.request.urlopen(url) as url:
    data = json.loads(url.read().decode())
    
# identifying type of python object
type(data)    

# reviewing the dictionairy's keys
data.keys()

# looking at the type of the data key
type(data['data'])

# previewing the length of items in the data list
len(data['data'])

# previewing the first row of our data
data['data'][0]

data['meta'].keys()

keys = data['meta']['view'].keys()

for key in keys:
    print(key)

# locating our columns (i.e. field names) and saving as a new variable called 'cols'
cols = data['meta']['view']['columns']
print('length of columns: {}'.format(len(cols)))

# previewing first five
cols[:5]

col_names = []

for col in cols:
    print(col['fieldName'])
    col_names.append(col['fieldName'])

# constructing a dataframe
df_json = pd.DataFrame(data['data'], columns=col_names)

df_json.head()

# removing columns that start with ":"
df_json = df_json.loc[:, ~df_json.columns.str.startswith(':')]

df_json.info()

# printing dimensions of data
rows, columns = df_json.shape
print('This dataset has {:,} rows and {:,} columns.'.format(rows, columns))

url = 'https://data.cityofnewyork.us/api/geospatial/tqmj-j8zm?method=export&format=Shapefile'

# reading in data as a geodataframe
gdf = gpd.read_file(url)

# printing the first five rows
gdf.head()

rows, columns = gdf.shape
print('This dataset has {:,} rows and {:,} columns.'.format(rows, columns))

# plotting boros
gdf.plot()

# nyc open data domain
socrata_domain = 'data.cityofnewyork.us' 

# building footprints dataset identifier
socrata_dataset_identifier = 'qb5r-6dgf' 

# The main class that interacts with the SODA API.     
client = Socrata(socrata_domain, None)

client.__dict__

print(
    "Domain: {domain:}\
    \nSession: {session:}\
    \nURI Prefix: {uri_prefix:}".format(**client.__dict__)
)

# retrieving data as a dictionary 
results = client.get(socrata_dataset_identifier, limit=1000)

# creating a dataframe from our dictionary
df_api = pd.DataFrame.from_dict(results)

# ending our API request
client.close()

# printing first five rows of data
df_api.head()

rows, columns = df_api.shape
print('This dataset has {:,} rows and {:,} columns.'.format(rows, columns))

# read data as a dataframe
df = pd.read_csv('data/sample-data.csv')

# previewing first five rows in data
df.head()

# writing files as a csv
df.to_csv('data/output.csv', index=False)

# listing items in data folder
%ls data/

# writing files as an excel file
df.to_excel('data/output.xlsx', index=False)

# listing items in data folder
%ls data/

# writing files as json
df.to_json('data/output.json')

# listing items in data folder
%ls data/

# read data as a geodataframe
gdf = gpd.read_file('data/sample-data.shp')

# previewing first five rows in data
gdf.head()

gdf.to_file('data/output.shp')

# listing items in data folder
%ls data/