How to store file on Bucket s3 with specific tree and then Zip it

Question

I'm working on this but amazon aws s3 is not my favorite part so I need some help with this:

import smartsheet
import time
import os
import json
import shutil
import zipfile
import boto3

#Initialisation Variables et Environment

token = api_token
ss = smartsheet.Smartsheet(token)
ssh = ss.Home.list_all_contents()
backupdir = "d:\\Backup"
if not os.path.exists(backupdir):
    os.mkdir(backupdir)
zipname = "SmartsheetBackup - " + time.strftime('%Y-%m-%d-%H-%M')
odir = "d:\\Backup\\SmartsheetBackup - " + time.strftime('%Y-%m-%d-%H-%M')
lsthome = json.loads(ssh.to_json())
sss = ss.Sheets.list_sheets(include_all=True) 
lstsheetsall = json.loads(sss.to_json())
rapport = open(odir + " Report.txt", "w+")
rapportname = zipname + " Report.txt"
lstsheets = []
workspaces = ss.Workspaces.list_workspaces(include_all=True)
lstworkspaces = json.loads(workspaces.to_json())
s3 = boto3.client('s3')
bucket = s3.Bucket(bucketname)

#Fin Initialisation

so actually with this setup we got:

• api_token which is the token smartsheet to backup data
• backupdir that would change with the bucket
• zipname the final name of the zip created
• ss,ssh,lsthom,lstsheetsall,lstsheets,workspaces,lstworkspaces are specific to my smartsheet-api var
• and the bucket s3 var with the bucket name

#Création Dossier de backup, rapport de backup

os.mkdir(odir)
rapport.write("---------- Starting Backup ----------\n\n")
rapport.write(time.strftime('%Y-%m-%d-%H-%M\n\n'))

here I'm creating a report file where I will put the timestamps etc...

#Backup des dossiers et sheets Inférieurs ou égales à F+2

print("Tool About to backup folder and sheets equals and less than f+2")
rapport.write("Start Backup Folders <= F+2\n\n")
for folder in lsthome['folders']:
    rapport.write("Create Folder : " + folder['name'] + "\n")
    os.mkdir(odir + "\\" + folder['name'])
    os.chdir(odir + "\\" + folder['name'])
    cdir = os.getcwd()
    for sheets in folder['sheets']:
        rapport.write("Download : " + sheets['name'] + "\n")
        ss.Sheets.get_sheet_as_excel(sheets['id'], os.getcwd())
        lstsheets.append(sheets['id'])
    if folder['folders'] == '[]':
        break
    else:
        for sfolder in folder['folders']:
            rapport.write("Create Folder : " + sfolder['name'] + "\n")
            os.mkdir(sfolder['name'])
            os.chdir(sfolder['name'])
            for ssheets in sfolder['sheets']:
                rapport.write("Download : " + ssheets['name'] + "\n")
                ss.Sheets.get_sheet_as_excel(ssheets['id'], os.getcwd())
                lstsheets.append(ssheets['id'])
rapport.write("Folder and sheets f+2 Backup OK \n\n")
print("Folders And Sheets Backup OK !")

#Fin De Backup des Dossiers

Then I Backup folder and file folder + 2 max with the right tree. (With s3 it should be faster)

#Initialisation du Backup des fichiers root

print("Tool About to backup Root Sheets")
rapport.write("Download Root Sheets \n")
for bsheets in lsthome['sheets']:
    os.chdir(odir)
    ss.Sheets.get_sheet_as_excel(bsheets['id'], os.getcwd())
    rapport.write("Download Rsheets : " + bsheets['name'] + "\n")
    lstsheets.append(bsheets['id'])
rapport.write("Root Sheets Backup OK \n\n")
print("Root Sheets Bakcup OK !")

#Fin de backup des fichier root

There is the backup of root file

#Backup des sheets restants dans SheetsSup

os.chdir(odir)
print("Tool About to backup file in more than f+2")
os.mkdir(odir + "\\SheetsSup")
os.chdir(odir + "\\SheetsSup")
rapport.write("\n\nBackup sheets over f+2\n\n")
if not(str(lstsheetsall['totalCount']) == str(len(lstsheets))):
    for sheetsall in lstsheetsall['data']:
        if not(sheetsall['id'] in lstsheets):
            rapport.write("Download : " + sheetsall['name'] + "\n")
            ss.Sheets.get_sheet_as_excel(sheetsall['id'], os.getcwd())
    os.chdir(odir)
rapport.write("Sheets Backup OK \n")
print("Sheets Backup Sup OK !")

#Fin backup Sheets over F+2

Here are the rest of the files (over folder + 2)

#Initialisation Backup Attachments

rapport.write("\n\nStart Backup Attachments\n\n")
print("Tool About to backup Attachments")
os.mkdir(odir + "\\Attachments")
os.chdir(odir + "\\Attachments")
for sheetsall in lstsheetsall['data']:
    DLA = ss.Attachments.list_all_attachments(sheetsall['id'], include_all=True)
    action = DLA.data
    for col in action:
        rapport.write("Download : " + col.name + "\n")
        attachurl = ss.Attachments.get_attachment(sheetsall['id'],col.id)
        ss.Attachments.download_attachment(attachurl, os.getcwd())
print("Download Attachments OK !")
rapport.write("Download Attach OK")

#Fin de backup attachments

Attachments on file and in folder smartsheet

#Initialisation Zip Dossier Backup + Fichier rapport

os.chdir(backupdir)
print("Zipping Backup")
rapport.write("Zipping Archive")
shutil.make_archive(zipname, 'zip', odir)
z = zipfile.ZipFile(zipname + ".zip", 'a')
rapport.write("\n\nRapport end at : " + time.strftime('%Y-%m-%d-%H-%M') + "\n")
rapport.write("----------- Ending Backup -----------")
rapport.close()
z.write(rapportname)
z.close()
print("Zipping Complete !")

#Fin de Zipping Dossier

#Néttoyage fichier et dossier

shutil.rmtree(odir)
os.remove(rapportname)

#Fin de Programme

Zipping of the folder root then del al files to just keep the zip.

Problem

And Now The Problem is how to make the same program but replacing my local storage that is my d:\folder to an s3 storage.

This Python program should be run with a lambda that schedule it to time to time (example : every day or 2 days). So No transition between PC in local.

Answer 1

You'll need to download to local storage, and then upload to S3.

Boto (https://github.com/boto/boto3) is the Python library for S3.