Introduction
Hi! Im trying to download application content-types from Microsoft TLU Delivery. Upon downloading, its always resulted with corrupted files. The downloaded files just have 1kb size, meanwhile the original ones have 200kb+. I already trying to use custom headers and cookies on the Download Module but its still the same. Here's my network connection that i get from chrome console :
Request URL: http://tlu.dl.delivery.mp.microsoft.com/filestreamingservice/files/f7ded4c4-f468-4bdc-96f0-03fd60c5ae81?p1=1604760658&p2=402&p3=2&p4=hcdsxul%2f8qkk8gjjjyomq0ki%2bxwjxulxxeninyf1jqrp3%2bsnzk%2fpwk4dgsnbkfzkzoor4%2bvaixilmxk6r%2bz6%2ba%3d%3d
Request Method: GET
Status Code: 200 OK
Remote Address: 8.241.131.254:80
Referrer Policy: strict-origin-when-cross-origin
#Response Header
Accept-Ranges: bytes
Age: 240667
Cache-Control: public, max-age=17280000
Connection: keep-alive
Content-Disposition: attachment; filename=Microsoft.Services.Store.Engagement_10.0.19011.0_x64__8wekyb3d8bbwe.Appx
Content-Length: 281728
Content-Type: application/octet-stream
Date: Sat, 21 Nov 2020 22:49:40 GMT
ETag: "cP6LQFFTB9bLaXCyN/YHr8kWbqM="
Expires: Wed, 09 Jun 2021 22:49:58 GMT
g: g
Last-Modified: Wed, 30 Jan 2019 14:28:15 GMT
MS-CorrelationId: c93312f7-dc52-41ef-b2d2-9294c1deda19
MS-CV: /24yhmSg0EivGZOl.0.3.8.2.1.0.0.22.2.6.2.1.1.0
MS-RequestId: 9378576c-a791-4636-93bf-7b8ef57c85a3
MSRegion: APAC
Server: Microsoft-IIS/10.0
X-AspNet-Version: 4.0.30319
X-AspNetMvc-Version: 5.2
x-ccc: SG
x-cid: 3
X-MS-Ref-OriginShield: Ref A: 4127806A5D794419ACF51D29DD883F50 Ref B: BAYEDGE0319 Ref C: 2019-01-30T14:32:41Z
X-MSEdge-Ref: Ref A: 6F76857D59174A4BA37F3076DE6671DC Ref B: SJCEDGE0406 Ref C: 2019-01-30T14:32:41Z
X-Powered-By: ASP.NET
#Request header
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
Accept-Encoding: gzip, deflate
Accept-Language: en-US,en;q=0.9,id;q=0.8,pt;q=0.7
Connection: keep-alive
Cookie: _ga=GA1.2.282065737.1604285823; MUID=07B92E6562A266AE0A83211366A260EF; AAMC_mscom_0=REGION%7C3; aam_uuid=33272659623977483402062764826775668547; mbox=PC#f8f94dce007c4a3c88788082771689c3.38_0#1668947906|session#af45a09527ce4486be93aff7c2a9adf1#1605704965; _cs_c=0; _cs_id=cd07a4b3-19b2-a8c7-95e4-7b8fd7c286f9.1605703107.1.1605703107.1605703107.1594299326.1639867107380.Lax.0; _uetvid=c457b5101e8a11eb956e2f93309fcf0c; IR_PI=f62c6991-299a-11eb-980c-0abbe301118c%7C1605789507507
DNT: 1
Host: tlu.dl.delivery.mp.microsoft.com
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36
#Query String
p1: 1604760658
p2: 402
p3: 2
p4: hcdsxul/8qkk8gjjjyomq0ki+xwjxulxxeninyf1jqrp3+snzk/pwk4dgsnbkfzkzoor4+vaixilmxk6r+z6+a==
Problematic
Here's several download module that i've try:
from clint import textui
from clint.textui import progress
import urllib.request
import requests
from tqdm import tqdm
from urllib.request import FancyURLopener
from tqdm import tqdm
class DownloadProgressBar(tqdm):
def update_to(self, b=1, bsize=1, tsize=None):
if tsize is not None:
self.total = tsize
self.update(b * bsize - self.n)
def CookDown(url, filename):
headers = {'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'}
cookies = {'_ga': 'GA1.2.282065737.1604285823', 'MUID': '07B92E6562A266AE0A83211366A260EF','aam_uuid': '33272659623977483402062764826775668547','AAMC_mscom_0': 'REGION%7C3','_cs_c': '0','_cs_id': 'cd07a4b3-19b2-a8c7-95e4-7b8fd7c286f9.1605703107.1.1605703107.1605703107.1594299326.1639867107380.Lax.0','_uetvid': 'c457b5101e8a11eb956e2f93309fcf0c','IR_PI': 'f62c6991-299a-11eb-980c-0abbe301118c%7C1605789507507','mbox': 'PC#f8f94dce007c4a3c88788082771689c3.38_0#1668947906|session#af45a09527ce4486be93aff7c2a9adf1#1605704965'}
try:
read = requests.get(url, headers=headers, cookies=cookies)
with open(filename, 'wb') as w:
for chunk in read.iter_content(chunk_size=None):
if chunk:
w.write(chunk)
print(filename + ' downloaded successfully!!!')
except urllib.request.URLError as e:
print("Error :%s" % e)
def Cookdone(url, filename):
r = requests.get(url, stream=True)
with open(filename, "wb") as f:
for data in tqdm(response.iter_content()):
f.write(data)
def Cookdown(url, filename):
#headers = {'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'}
#cookies = {'_ga': 'GA1.2.282065737.1604285823', 'MUID': '07B92E6562A266AE0A83211366A260EF','aam_uuid': '33272659623977483402062764826775668547','AAMC_mscom_0': 'REGION%7C3','_cs_c': '0','_cs_id': 'cd07a4b3-19b2-a8c7-95e4-7b8fd7c286f9.1605703107.1.1605703107.1605703107.1594299326.1639867107380.Lax.0','_uetvid': 'c457b5101e8a11eb956e2f93309fcf0c','IR_PI': 'f62c6991-299a-11eb-980c-0abbe301118c%7C1605789507507','mbox': 'PC#f8f94dce007c4a3c88788082771689c3.38_0#1668947906|session#af45a09527ce4486be93aff7c2a9adf1#1605704965'}
import shutil
req = requests.get(url, stream=True)
print(req.headers)
with open(filename, "wb") as f:
req.raw.decode_content = False
shutil.copyfileobj(req.raw, f)
def download(url, path):
opener = FancyURLopener({})
opener.version = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'
opener.retrieve(url, path)
def down(url, filename):
proxy = ProxyHandler({})
opener = build_opener(proxy)
opener.addheaders = [('User-Agent','Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.1 Safari/603.1.30')]
install_opener(opener)
urlretrieve(url, filename)
def DownLoad(url, filename):
headers = {'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'}
r = requests.get(url, allow_redirects=True, headers=headers)
with open(filename, 'wb') as f:
for chunk in r.iter_content(1024):
f.write(chunk)
def DoneLoad():
headers = {'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'}
r = requests.get(url, allow_redirects=True, headers=headers)
with open(filename, 'wb') as f:
total_length = int(r.headers.get('content-length'))
for chunk in progress.bar(r.iter_content(chunk_size=1024), expected_size=(total_length/1024) + 1):
if chunk:
f.write(chunk)
f.flush()
def Download(url, path):
hdr = { 'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36' }
req = urllib.request.Request(url, headers=hdr)
urllib.request.urlretrieve(req, path)
def downLoader(url, output_path):
with DownloadProgressBar(unit='B', unit_scale=True,
miniters=1, desc=url.split('/')[-1]) as t:
opener = urllib.request.build_opener()
opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36')]
urllib.request.install_opener(opener)
urllib.request.urlretrieve(url, filename=output_path, reporthook=t.update_to)
def downLoad(url, output_path):
hdr = { 'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36' }
r = requests.get(url, headers=hdr, verify=False, stream=True)
with open(output_path, 'wb') as f:
for chunk in r.iter_content(chunk_size=None):
f.write(chunk)
Here's piece of code to execute every each module :
CookDown('http://tlu.dl.delivery.mp.microsoft.com/filestreamingservice/files/f7ded4c4-f468-4bdc-96f0-03fd60c5ae81?p1=1604760658&p2=402&p3=2&p4=hcdsxul%2f8qkk8gjjjyomq0ki%2bxwjxulxxeninyf1jqrp3%2bsnzk%2fpwk4dgsnbkfzkzoor4%2bvaixilmxk6r%2bz6%2ba%3d%3d','Microsoft.Services.Store.Engagement_10.0.19011.0_x64__8wekyb3d8bbwe.appx')
Using each module, resulted on the same corrupted file. Any answers will hugely appreciated because im stuck and doesnt have any ideas anymore. Please stackoverflow gods...Come here :D