X-Git-Url: https://git.ralfj.de/git-mirror.git/blobdiff_plain/503462ba4433cef0cf10deaf9595a431157832ee..2b00eaa147818ccfd68a1466d28c41995accef7d:/git_mirror.py diff --git a/git_mirror.py b/git_mirror.py index 9b5b528..12075bf 100644 --- a/git_mirror.py +++ b/git_mirror.py @@ -1,7 +1,34 @@ -import sys, os, subprocess -import configparser, itertools, json, re +# Copyright (c) 2015, Ralf Jung +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#============================================================================== +import sys, os, os.path, subprocess +import configparser, itertools, re +import hmac, hashlib import email.mime.text, email.utils, smtplib +mail_sender = "null@localhost" +config_file = os.path.join(os.path.dirname(__file__), 'git-mirror.conf') + class GitCommand: def __getattr__(self, name): def call(*args, capture_stderr = False, check = True): @@ -9,12 +36,12 @@ class GitCommand: If is true, throw an exception of the process fails with non-zero exit code. Otherwise, do not. In any case, return a pair of the captured output and the exit code.''' cmd = ["git", name.replace('_', '-')] + list(args) - with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT if capture_stderr else None) as p: + with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT if capture_stderr else sys.stderr) as p: (stdout, stderr) = p.communicate() assert stderr is None code = p.returncode if check and code: - raise Exception("Error running {0}: Non-zero exit code".format(cmd)) + raise Exception("Error running {}: Non-zero exit code".format(cmd)) return (stdout.decode('utf-8').strip('\n'), code) return call @@ -27,45 +54,38 @@ def git_is_forced_update(oldsha, newsha): assert code in (0, 1) return False if code == 0 else True # if oldsha is an ancestor of newsha, then this was a "good" (non-forced) update -def read_config(fname, defSection = 'DEFAULT'): +def read_config(defSection = 'DEFAULT'): '''Reads a config file that may have options outside of any section.''' config = configparser.ConfigParser() - with open(fname) as file: + with open(config_file) as file: stream = itertools.chain(("["+defSection+"]\n",), file) config.read_file(stream) return config -def send_mail(subject, text, receivers, sender='post+webhook@ralfj.de', replyTo=None): - assert isinstance(receivers, list) - if not len(receivers): return # nothing to do +def send_mail(subject, text, recipients, sender, replyTo = None): + assert isinstance(recipients, list) + if not len(recipients): return # nothing to do # construct content msg = email.mime.text.MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8') msg['Subject'] = subject msg['Date'] = email.utils.formatdate(localtime=True) msg['From'] = sender - msg['To'] = ', '.join(receivers) + msg['To'] = ', '.join(recipients) if replyTo is not None: msg['Reply-To'] = replyTo # put into envelope and send s = smtplib.SMTP('localhost') - s.sendmail(sender, receivers, msg.as_string()) + s.sendmail(sender, recipients, msg.as_string()) s.quit() -def get_github_payload(): - '''Reeturn the github-style JSON encoded payload (as if we were called as a github webhook)''' - try: - data = sys.stdin.buffer.read() - data = json.loads(data.decode('utf-8')) - return data - except: - return {} # nothing read - class Repo: def __init__(self, name, conf): '''Creates a repository from a section of the git-mirror configuration file''' self.name = name self.local = conf['local'] self.owner = conf['owner'] # email address to notify in case of problems + self.hmac_secret = conf['hmac-secret'].encode('utf-8') if 'hmac-secret' in conf else None + self.deploy_key = conf['deploy-key'] # the SSH ky used for authenticating against remote hosts self.mirrors = {} # maps mirrors to their URLs mirror_prefix = 'mirror-' for name in filter(lambda s: s.startswith(mirror_prefix), conf.keys()): @@ -73,7 +93,14 @@ class Repo: self.mirrors[mirror] = conf[name] def mail_owner(self, msg): - send_mail("git-mirror {0}".format(self.name), msg, [self.owner]) + global mail_sender + send_mail("git-mirror {}".format(self.name), msg, recipients = [self.owner], sender = mail_sender) + + def compute_hmac(self, data): + assert self.hmac_secret is not None + h = hmac.new(self.hmac_secret, digestmod = hashlib.sha1) + h.update(data) + return h.hexdigest() def find_mirror_by_url(self, match_urls): for mirror, url in self.mirrors.items(): @@ -81,27 +108,37 @@ class Repo: return mirror return None - def update_mirrors(self, ref, oldsha, newsha, except_mirrors = [], suppress_stderr = False): + def setup_env(self): + '''Setup the environment to work with this repository''' + os.chdir(self.local) + ssh_set_ident = os.path.join(os.path.dirname(__file__), 'ssh-set-ident.sh') + os.putenv('GIT_SSH', ssh_set_ident) + ssh_ident = os.path.join(os.path.expanduser('~/.ssh'), self.deploy_key) + os.putenv('GIT_MIRROR_SSH_IDENT', ssh_ident) + + def update_mirrors(self, ref, oldsha, newsha): '''Update the from to on all mirrors. The update must already have happened locally.''' assert len(oldsha) == 40 and len(newsha) == 40, "These are not valid SHAs." - os.chdir(self.local) + source_mirror = os.getenv("GIT_MIRROR_SOURCE") # in case of a self-call via the hooks, we can skip one of the mirrors + self.setup_env() # check for a forced update is_forced = newsha != git_nullsha and oldsha != git_nullsha and git_is_forced_update(oldsha, newsha) # tell all the mirrors for mirror in self.mirrors: - if mirror in except_mirrors: + if mirror == source_mirror: continue + sys.stdout.write("Updating mirror {}\n".format(mirror)); sys.stdout.flush() # update this mirror if is_forced: # forcibly update ref remotely (someone already did a force push and hence accepted data loss) - git.push('--force', self.mirrors[mirror], newsha+":"+ref, capture_stderr = suppress_stderr) + git.push('--force', self.mirrors[mirror], newsha+":"+ref) else: # nicely update ref remotely (this avoids data loss due to race conditions) - git.push(self.mirrors[mirror], newsha+":"+ref, capture_stderr = suppress_stderr) + git.push(self.mirrors[mirror], newsha+":"+ref) def update_ref_from_mirror(self, ref, oldsha, newsha, mirror, suppress_stderr = False): '''Update the local version of this to what's currently on the given . and are checked. Then update all the other mirrors.''' - os.chdir(self.local) + self.setup_env() url = self.mirrors[mirror] # first check whether the remote really is at newsha remote_state, code = git.ls_remote(url, ref) @@ -109,19 +146,20 @@ class Repo: remote_sha = remote_state.split()[0] else: remote_sha = git_nullsha - assert newsha == remote_sha, "Someone lied about the new SHA, which should be {0}.".format(newsha) + assert newsha == remote_sha, "Someone lied about the new SHA, which should be {}.".format(newsha) # locally, we have to be at oldsha or newsha (the latter can happen if we already got this update, e.g. if it originated from us) local_state, code = git.show_ref(ref, check=False) if code == 0: local_sha = local_state.split()[0] else: if len(local_state): - raise Exception("Something went wrong getting the local state of {0}.".format(ref)) + raise Exception("Something went wrong getting the local state of {}.".format(ref)) local_sha = git_nullsha - assert local_sha in (oldsha, newsha), "Someone lied about the old SHA." + # some sanity checking, but deal gracefully with new branches appearing + assert local_sha in (git_nullsha, oldsha, newsha), "Someone lied about the old SHA: Local ({}) is neither old ({}) nor new ({})".format(local_sha, oldsha, newsha) # if we are already at newsha locally, we also ran the local hooks, so we do not have to do anything if local_sha == newsha: - return + return "Local repository is already up-to-date." # update local state from local_sha to newsha. if newsha != git_nullsha: # We *could* now fetch the remote ref and immediately update the local one. However, then we would have to @@ -136,8 +174,15 @@ class Repo: # ref does not exist anymore. delete it. assert local_sha != git_nullsha, "Why didn't we bail out earlier if there is nothing to do...?" git.update_ref("-d", ref, local_sha) # this checks that the old value is still local_sha - # update all the mirrors - self.update_mirrors(ref, oldsha, newsha, [mirror], suppress_stderr) + # Now run the post-receive hooks. This will *also* push the changes to all mirrors, as we + # are one of these hooks! + os.putenv("GIT_MIRROR_SOURCE", mirror) # tell ourselves which repo we do *not* have to update + with subprocess.Popen(['hooks/post-receive'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as p: + (stdout, stderr) = p.communicate("{} {} {}\n".format(oldsha, newsha, ref).encode('utf-8')) + stdout = stdout.decode('utf-8') + if p.returncode: + raise Exception("post-receive git hook terminated with non-zero exit code {}:\n{}".format(p.returncode, stdout)) + return stdout def find_repo_by_directory(repos, dir): for (name, repo) in repos.items(): @@ -146,8 +191,10 @@ def find_repo_by_directory(repos, dir): return None def load_repos(): - conffile = os.path.join(os.path.dirname(__file__), 'git-mirror.conf') - conf = read_config(conffile) + global mail_sender + conf = read_config() + mail_sender = conf['DEFAULT']['mail-sender'] + repos = {} for name, section in conf.items(): if name != 'DEFAULT':