[MediaWiki-commits] [Gerrit] pywikibot/core[master]: download_dump: Add -revision parameter

[Rafidaslam (Code Review)]

Rafidaslam has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/401091 )

Change subject: download_dump: Add -revision parameter
......................................................................

download_dump: Add -revision parameter

Add `-revision` parameter in
`scripts/maintenance/download_dump.py` which used for
specifying the revision of the file dump that wanna be
downloaded.

Bug: T183789
Change-Id: I3e7e22f3f5f35e71e3793f8f5b568c7e97aa4b1c
---
M scripts/maintenance/download_dump.py
1 file changed, 24 insertions(+), 3 deletions(-)


git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core refs/changes/91/401091/1

diff --git a/scripts/maintenance/download_dump.py b/scripts/maintenance/download_dump.py
index d3a5123..05842c1 100644
--- a/scripts/maintenance/download_dump.py
+++ b/scripts/maintenance/download_dump.py
@@ -9,6 +9,8 @@

-storepath:# The stored file's path.

+ -revision:# The revision date of the dump (default to `latest`)
+
"""
#
# (C) Pywikibot team, 2017
@@ -38,6 +40,7 @@
'wikiname': '',
'filename': '',
'storepath': './',
+ 'revision': 'latest',
}

def __init__(self, **kwargs):
@@ -61,8 +64,11 @@
"""Run bot."""
pywikibot.output('Downloading dump from ' + self.getOption('wikiname'))

- download_filename = self.getOption('wikiname') + \
- '-latest-' + self.getOption('filename')
+ download_filename = '{wiki_name}-{revision}-{filename}'.format(
+ wiki_name=self.getOption('wikiname'),
+ revision=self.getOption('revision'),
+ filename=self.getOption('filename')
+ )
file_storepath = os.path.join(
self.getOption('storepath'), download_filename)

@@ -78,7 +84,8 @@
else:
url = 'https://dumps.wikimedia.org/' + \
os.path.join(self.getOption('wikiname'),
- 'latest', download_filename)
+ self.getOption('revision'),
+ download_filename)
pywikibot.output('Downloading file from ' + url)
response = fetch(url, stream=True)
if response.status == 200:
@@ -89,6 +96,15 @@
except IOError:
pywikibot.exception()
return False
+ elif response.status == 404:
+ pywikibot.output(('File with wikiname={wikiname}, '
+ 'revision={revision}, '
+ 'filename={filename} doesn\'t '
+ 'available in the Wikimedia Dumps').format(
+ wikiname=self.getOption('wikiname'),
+ revision=self.getOption('revision'),
+ filename=self.getOption('filename')))
+ return
else:
return

@@ -121,6 +137,11 @@
opts[option] = os.path.abspath(value) or pywikibot.input(
'Enter the store path: ')
continue
+ elif option == 'revision':
+ opts[option] = value or pywikibot.input(
+ 'Enter the revision of the dump: ')
+ continue
+
unknown_args += [arg]

missing = []
--
To view, visit https://gerrit.wikimedia.org/r/401091
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I3e7e22f3f5f35e71e3793f8f5b568c7e97aa4b1c
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Rafidaslam <***@gmail.com>

[jenkins-bot (Code Review)]

jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/401091 )

Change subject: download_dump: Add -revision parameter
......................................................................


download_dump: Add -revision parameter

Add `-revision` parameter in
`scripts/maintenance/download_dump.py` which used for
specifying the revision of the file dump that wanna be
downloaded.

Bug: T183789
Change-Id: I3e7e22f3f5f35e71e3793f8f5b568c7e97aa4b1c
---
M scripts/maintenance/download_dump.py
1 file changed, 29 insertions(+), 4 deletions(-)

Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved



diff --git a/scripts/maintenance/download_dump.py b/scripts/maintenance/download_dump.py
index c55940b..853d33f 100644
--- a/scripts/maintenance/download_dump.py
+++ b/scripts/maintenance/download_dump.py
@@ -9,6 +9,9 @@

-storepath:# The stored file's path.

+ -revision:# The revision date of the dump (default to `latest`)
+ formatted as YYYYMMDD.
+
"""
#
# (C) Pywikibot team, 2017
@@ -55,6 +58,7 @@
'wikiname': '',
'filename': '',
'storepath': './',
+ 'revision': 'latest',
}

def __init__(self, **kwargs):
@@ -78,8 +82,11 @@
"""Run bot."""
pywikibot.output('Downloading dump from ' + self.getOption('wikiname'))

- download_filename = self.getOption('wikiname') + \
- '-latest-' + self.getOption('filename')
+ download_filename = '{wiki_name}-{revision}-{filename}'.format(
+ wiki_name=self.getOption('wikiname'),
+ revision=self.getOption('revision'),
+ filename=self.getOption('filename')
+ )
temp_filename = download_filename + '-' + \
binascii.b2a_hex(urandom(8)).decode('ascii') + '.part'

@@ -104,14 +111,27 @@
remove(file_final_storepath)
symlink(toolforge_dump_filepath, file_current_storepath)
else:
- url = 'https://dumps.wikimedia.org/{0}/latest/{1}'.format(
- self.getOption('wikiname'), download_filename)
+ url = 'https://dumps.wikimedia.org/{0}/{1}/{2}'.format(
+ self.getOption('wikiname'),
+ self.getOption('revision'),
+ download_filename)
pywikibot.output('Downloading file from ' + url)
response = fetch(url, stream=True)
if response.status == 200:
with open(file_current_storepath, 'wb') as result_file:
for data in response.data.iter_content(100 * 1024):
result_file.write(data)
+ elif response.status == 404:
+ pywikibot.output(
+ 'File with name "{filename}", '
+ 'from revision "{revision}", '
+ 'and wiki "{wikiname}" ({url}) isn\'t '
+ 'available in the Wikimedia Dumps'.format(
+ filename=self.getOption('filename'),
+ revision=self.getOption('revision'),
+ url=url,
+ wikiname=self.getOption('wikiname')))
+ return
else:
return
# Rename the temporary file to the target file
@@ -165,6 +185,11 @@
opts[option] = os.path.abspath(value) or pywikibot.input(
'Enter the store path: ')
continue
+ elif option == 'revision':
+ opts[option] = value or pywikibot.input(
+ 'Enter the revision of the dump: ')
+ continue
+
unknown_args += [arg]

missing = []
--
To view, visit https://gerrit.wikimedia.org/r/401091
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: I3e7e22f3f5f35e71e3793f8f5b568c7e97aa4b1c
Gerrit-PatchSet: 5
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Rafidaslam <***@gmail.com>
Gerrit-Reviewer: Framawiki <***@tools.wmflabs.org>
Gerrit-Reviewer: John Vandenberg <***@gmail.com>
Gerrit-Reviewer: Xqt <***@gno.de>
Gerrit-Reviewer: Zhuyifei1999 <***@gmail.com>
Gerrit-Reviewer: Zoranzoki21 <***@gmail.com>
Gerrit-Reviewer: jenkins-bot <>