352 lines
12 KiB
Python
Executable File
352 lines
12 KiB
Python
Executable File
#!/usr/bin/python3
|
|
# -*- coding: utf-8 -*-
|
|
"""Process new requests to download per-package data"""
|
|
# Copyright (C) 2012 Canonical Ltd
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of version 3 of the GNU General Public License as
|
|
# published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
|
|
import glob
|
|
import os
|
|
import sys
|
|
import subprocess
|
|
import traceback
|
|
import debian.deb822
|
|
import string
|
|
import debconf
|
|
from datetime import datetime
|
|
|
|
import apt_pkg
|
|
|
|
# avoid hanging forever (LP: #1243090)
|
|
import socket
|
|
socket.setdefaulttimeout(60)
|
|
|
|
|
|
DATADIR = "/usr/share/package-data-downloads/"
|
|
STAMPDIR = "/var/lib/update-notifier/package-data-downloads/"
|
|
NOTIFIER_SOURCE_FILE = \
|
|
"/usr/share/update-notifier/package-data-downloads-failed"
|
|
NOTIFIER_FILE = "/var/lib/update-notifier/user.d/data-downloads-failed"
|
|
NOTIFIER_PERMANENT_SOURCE_FILE = NOTIFIER_SOURCE_FILE + '-permanently'
|
|
NOTIFIER_PERMANENT_FILE = NOTIFIER_FILE + '-permanently'
|
|
|
|
failures = []
|
|
permanent_failures = []
|
|
|
|
|
|
def create_or_update_stampfile(file):
|
|
"""Create or update the indicated stampfile, and remove failure flags"""
|
|
|
|
try:
|
|
with open(file, 'w'):
|
|
pass
|
|
# Ignore errors
|
|
except Exception:
|
|
traceback.print_exc(file=sys.stderr)
|
|
|
|
os.utime(file, None)
|
|
|
|
for ext in ('.failed', '.permanent-failure'):
|
|
if os.path.exists(file + ext):
|
|
os.unlink(file + ext)
|
|
|
|
|
|
def mark_hook_failed(hook_name, permanent=False):
|
|
"""Create a stampfile recording that a hook failed
|
|
|
|
We create separate stampfiles for failed hooks so we can
|
|
keep track of how long the hook has been failing and if the failure
|
|
should be considered permanent."""
|
|
|
|
if permanent:
|
|
filename = hook_name + '.permanent-failure'
|
|
else:
|
|
filename = hook_name + '.failed'
|
|
|
|
failure_file = os.path.join(STAMPDIR, filename)
|
|
try:
|
|
with open(failure_file, 'w'):
|
|
pass
|
|
|
|
# Ignore errors
|
|
except Exception:
|
|
traceback.print_exc(file=sys.stderr)
|
|
|
|
for ext in ('', '.failed', '.permanent-failure'):
|
|
stampfile = hook_name + ext
|
|
if filename != stampfile \
|
|
and os.path.exists(os.path.join(STAMPDIR, stampfile)):
|
|
os.unlink(os.path.join(STAMPDIR, stampfile))
|
|
|
|
|
|
def hook_is_permanently_failed(hook_name):
|
|
"""Check if this hook has been marked as permanently failing.
|
|
|
|
If so, don't raise any more errors about it."""
|
|
|
|
failure_file = os.path.join(STAMPDIR, hook_name + '.permanent-failure')
|
|
return os.path.exists(failure_file)
|
|
|
|
|
|
def hook_aged_out(hook_name):
|
|
"""Check if this hook has been failing consistently for >= 3 days"""
|
|
|
|
failure_file = os.path.join(STAMPDIR, hook_name + '.failed')
|
|
try:
|
|
hook_date = datetime.fromtimestamp(os.stat(failure_file).st_ctime)
|
|
cur_time = datetime.now()
|
|
d = cur_time - hook_date
|
|
if d.days >= 3:
|
|
return True
|
|
except OSError:
|
|
pass
|
|
except Exception:
|
|
traceback.print_exc(file=sys.stderr)
|
|
return False
|
|
|
|
|
|
def record_failure(hook):
|
|
"""Record that the named hook has failed"""
|
|
if hook_aged_out(hook):
|
|
permanent_failures.append(hook)
|
|
else:
|
|
failures.append(hook)
|
|
|
|
|
|
def existing_permanent_failures():
|
|
"""Return the list of all previously recorded permanent failures"""
|
|
|
|
files = glob.glob(os.path.join(STAMPDIR, "*.permanent-failure"))
|
|
return [os.path.splitext(os.path.basename(path))[0] for path in files]
|
|
|
|
|
|
def trigger_update_notifier(failures, permanent=False):
|
|
"""Tell update-notifier that there were failed packages"""
|
|
|
|
try:
|
|
if permanent:
|
|
with open(NOTIFIER_PERMANENT_SOURCE_FILE, 'r',
|
|
encoding='utf-8') as f:
|
|
input = f.read()
|
|
output_file = open(NOTIFIER_PERMANENT_FILE, 'w', encoding='utf-8')
|
|
else:
|
|
with open(NOTIFIER_SOURCE_FILE, 'r', encoding='utf-8') as f:
|
|
input = f.read()
|
|
output_file = open(NOTIFIER_FILE, 'w', encoding='utf-8')
|
|
except Exception:
|
|
# Things failed and we can't even notify about it. Break the
|
|
# trigger so that there's some error propagation, even if not
|
|
# the most pleasant sort.
|
|
traceback.print_exc(file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
packages = [os.path.basename(failure) for failure in failures]
|
|
output_file.write(
|
|
string.Template(input).substitute(
|
|
{'packages': ", ".join(packages)}))
|
|
output_file.close()
|
|
|
|
|
|
def get_hook_file_names():
|
|
res = []
|
|
for relfile in os.listdir(DATADIR):
|
|
# ignore files ending in .dpkg-*
|
|
if (os.path.splitext(relfile)[1]
|
|
and os.path.splitext(relfile)[1].startswith(".dpkg")):
|
|
continue
|
|
res.append(relfile)
|
|
return res
|
|
|
|
|
|
# we use apt-helper here as this gives us the exact same proxy behavior
|
|
# as apt-get itself (environment/apt-config proxy settings/autodiscover)
|
|
def download_file(uri, sha256_hashsum):
|
|
"""Download a URI and checks the given hashsum using apt-helper
|
|
|
|
Returns: path to the downloaded file or None
|
|
"""
|
|
download_dir = os.path.join(STAMPDIR, "partial")
|
|
dest_file = os.path.join(download_dir, os.path.basename(uri))
|
|
try:
|
|
with open(dest_file, 'rb') as dest_file_obj:
|
|
# apt_pkg can directly hash from file obj, let's use that instead
|
|
# of hashlib
|
|
real_sha256 = apt_pkg.sha256sum(dest_file_obj)
|
|
if real_sha256 == sha256_hashsum:
|
|
return dest_file
|
|
else:
|
|
os.remove(dest_file)
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
ret = subprocess.call(
|
|
["/usr/lib/apt/apt-helper",
|
|
"download-file", uri, dest_file, "SHA256:" + sha256_hashsum])
|
|
if ret != 0:
|
|
if os.path.exists(dest_file):
|
|
os.remove(dest_file)
|
|
return None
|
|
return dest_file
|
|
|
|
|
|
def print_maybe(*args, **kwargs):
|
|
"""Version of print() that ignores failure"""
|
|
try:
|
|
print(*args, **kwargs)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def process_download_requests():
|
|
"""Process requests to download package data files
|
|
|
|
Iterate over /usr/share/package-data-downloads and download any
|
|
package data specified in the contained file, then hand off to
|
|
the indicated handler for further processing.
|
|
|
|
Successful downloads are recorded in
|
|
/var/lib/update-notifier/package-data-downloads to avoid unnecessary
|
|
repeat handling.
|
|
|
|
Failed downloads are reported to the user via the
|
|
update-notifier interface."""
|
|
|
|
# Iterate through all the available hooks. If we get a failure
|
|
# processing any of them (download failure, checksum failure, or
|
|
# failure to run the hook script), record it but continue processing
|
|
# the rest of the hooks since some of them may succeed.
|
|
for relfile in get_hook_file_names():
|
|
|
|
stampfile = os.path.join(STAMPDIR, relfile)
|
|
file = os.path.join(DATADIR, relfile)
|
|
try:
|
|
if not os.path.exists(NOTIFIER_FILE) and \
|
|
not os.path.exists(NOTIFIER_PERMANENT_FILE):
|
|
hook_date = os.stat(file).st_mtime
|
|
stamp_date = os.stat(stampfile).st_mtime
|
|
if hook_date < stamp_date:
|
|
continue
|
|
elif os.path.exists(stampfile):
|
|
continue
|
|
|
|
except Exception as e:
|
|
if not isinstance(e, OSError):
|
|
traceback.print_exc(file=sys.stderr)
|
|
|
|
hook = debian.deb822.Deb822()
|
|
files = []
|
|
sums = []
|
|
for para in hook.iter_paragraphs(open(file)):
|
|
if 'Script' in para:
|
|
if not files:
|
|
record_failure(relfile)
|
|
break
|
|
command = [para['Script']]
|
|
|
|
if 'Should-Download' in para:
|
|
db = debconf.DebconfCommunicator('update-notifier')
|
|
try:
|
|
should = db.get(para['Should-Download'])
|
|
if should == "false":
|
|
# Do nothing with this file.
|
|
break
|
|
except (DebconfError, KeyError):
|
|
pass
|
|
finally:
|
|
db.shutdown()
|
|
|
|
print_maybe("%s: processing..." % (relfile))
|
|
|
|
# Download each file and verify the sum
|
|
try:
|
|
downloaded = set()
|
|
for i in range(len(files)):
|
|
print_maybe("%s: downloading %s" % (relfile, files[i]))
|
|
dest_file = download_file(files[i], sums[i])
|
|
if dest_file:
|
|
command.append(dest_file)
|
|
downloaded.add(dest_file)
|
|
else:
|
|
record_failure(relfile)
|
|
break
|
|
if relfile in failures + permanent_failures:
|
|
break
|
|
|
|
sys.stdout.flush()
|
|
result = subprocess.call(command)
|
|
if result:
|
|
# There's no sense redownloading if the script fails
|
|
permanent_failures.append(relfile)
|
|
else:
|
|
create_or_update_stampfile(stampfile)
|
|
# cleanup
|
|
for f in downloaded:
|
|
os.remove(f)
|
|
break
|
|
except Exception:
|
|
traceback.print_exc(file=sys.stderr)
|
|
|
|
record_failure(relfile)
|
|
# The 'script' is always the last stanza
|
|
break
|
|
|
|
# Not in a 'script' stanza, so we should have some urls
|
|
try:
|
|
files.append(para['Url'])
|
|
sums.append(para['Sha256'])
|
|
except Exception as e:
|
|
print_maybe("%s: Error processing!" % (relfile))
|
|
if not isinstance(e, KeyError):
|
|
traceback.print_exc(file=sys.stderr)
|
|
record_failure(relfile)
|
|
break
|
|
|
|
previous_failures = existing_permanent_failures()
|
|
|
|
# We only report about "permanent" failures when there are new ones,
|
|
# but we want the whole list of permanently-failing hooks so when
|
|
# we clobber the update-notifier file we don't lose information the
|
|
# user may not have seen yet
|
|
if permanent_failures:
|
|
new_failures = False
|
|
for failure in permanent_failures:
|
|
if failure not in previous_failures:
|
|
mark_hook_failed(failure, permanent=True)
|
|
previous_failures.append(failure)
|
|
new_failures = True
|
|
if new_failures:
|
|
trigger_update_notifier(previous_failures, permanent=True)
|
|
# 2016-09-19 14:36 reset the list of permanent_failures as it caused
|
|
# tests not to be idempotent
|
|
permanent_failures.clear()
|
|
if not previous_failures and os.path.exists(NOTIFIER_PERMANENT_FILE):
|
|
os.unlink(NOTIFIER_PERMANENT_FILE)
|
|
|
|
# Filter out new failure reports for permanently-failed packages
|
|
our_failures = [x for x in failures if x not in previous_failures]
|
|
# 2016-09-19 14:36 reset the list of permanent_failures as it caused
|
|
# tests not to be idempotent
|
|
failures.clear()
|
|
|
|
if our_failures:
|
|
for failure in our_failures:
|
|
mark_hook_failed(failure)
|
|
trigger_update_notifier(our_failures)
|
|
elif os.path.exists(NOTIFIER_FILE):
|
|
os.unlink(NOTIFIER_FILE)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
process_download_requests()
|