X-Git-Url: https://git.ralfj.de/git-mirror.git/blobdiff_plain/b13f671882eb0d0e302860144103870f38f4a062..301520c0234eb50350361fac263bcebd2db139c2:/git_mirror.py?ds=sidebyside diff --git a/git_mirror.py b/git_mirror.py index ab8699e..0fe699e 100644 --- a/git_mirror.py +++ b/git_mirror.py @@ -21,10 +21,13 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #============================================================================== -import sys, os, subprocess -import configparser, itertools, json, re +import sys, os, os.path, subprocess +import configparser, itertools, re +import hmac, hashlib import email.mime.text, email.utils, smtplib +mail_sender = "null@localhost" + class GitCommand: def __getattr__(self, name): def call(*args, capture_stderr = False, check = True): @@ -32,7 +35,7 @@ class GitCommand: If is true, throw an exception of the process fails with non-zero exit code. Otherwise, do not. In any case, return a pair of the captured output and the exit code.''' cmd = ["git", name.replace('_', '-')] + list(args) - with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT if capture_stderr else None) as p: + with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT if capture_stderr else sys.stderr) as p: (stdout, stderr) = p.communicate() assert stderr is None code = p.returncode @@ -58,37 +61,30 @@ def read_config(fname, defSection = 'DEFAULT'): config.read_file(stream) return config -def send_mail(subject, text, receivers, sender='post+webhook@ralfj.de', replyTo=None): - assert isinstance(receivers, list) - if not len(receivers): return # nothing to do +def send_mail(subject, text, recipients, sender, replyTo = None): + assert isinstance(recipients, list) + if not len(recipients): return # nothing to do # construct content msg = email.mime.text.MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8') msg['Subject'] = subject msg['Date'] = email.utils.formatdate(localtime=True) msg['From'] = sender - msg['To'] = ', '.join(receivers) + msg['To'] = ', '.join(recipients) if replyTo is not None: msg['Reply-To'] = replyTo # put into envelope and send s = smtplib.SMTP('localhost') - s.sendmail(sender, receivers, msg.as_string()) + s.sendmail(sender, recipients, msg.as_string()) s.quit() -def get_github_payload(): - '''Reeturn the github-style JSON encoded payload (as if we were called as a github webhook)''' - try: - data = sys.stdin.buffer.read() - data = json.loads(data.decode('utf-8')) - return data - except: - return {} # nothing read - class Repo: def __init__(self, name, conf): '''Creates a repository from a section of the git-mirror configuration file''' self.name = name self.local = conf['local'] self.owner = conf['owner'] # email address to notify in case of problems + self.hmac_secret = conf['hmac-secret'].encode('utf-8') + self.deploy_key = conf['deploy-key'] # the SSH ky used for authenticating against remote hosts self.mirrors = {} # maps mirrors to their URLs mirror_prefix = 'mirror-' for name in filter(lambda s: s.startswith(mirror_prefix), conf.keys()): @@ -96,7 +92,13 @@ class Repo: self.mirrors[mirror] = conf[name] def mail_owner(self, msg): - send_mail("git-mirror {0}".format(self.name), msg, [self.owner]) + global mail_sender + send_mail("git-mirror {0}".format(self.name), msg, recipients = [self.owner], sender = mail_sender) + + def compute_hmac(self, data): + h = hmac.new(self.hmac_secret, digestmod = hashlib.sha1) + h.update(data) + return h.hexdigest() def find_mirror_by_url(self, match_urls): for mirror, url in self.mirrors.items(): @@ -104,27 +106,37 @@ class Repo: return mirror return None - def update_mirrors(self, ref, oldsha, newsha, except_mirrors = [], suppress_stderr = False): + def setup_env(self): + '''Setup the environment to work with this repository''' + os.chdir(self.local) + ssh_set_ident = os.path.join(os.path.dirname(__file__), 'ssh-set-ident.sh') + os.putenv('GIT_SSH', ssh_set_ident) + ssh_ident = os.path.join(os.path.expanduser('~/.ssh'), self.deploy_key) + os.putenv('GIT_MIRROR_SSH_IDENT', ssh_ident) + + def update_mirrors(self, ref, oldsha, newsha): '''Update the from to on all mirrors. The update must already have happened locally.''' assert len(oldsha) == 40 and len(newsha) == 40, "These are not valid SHAs." - os.chdir(self.local) + source_mirror = os.getenv("GIT_MIRROR_SOURCE") # in case of a self-call via the hooks, we can skip one of the mirrors + self.setup_env() # check for a forced update is_forced = newsha != git_nullsha and oldsha != git_nullsha and git_is_forced_update(oldsha, newsha) # tell all the mirrors for mirror in self.mirrors: - if mirror in except_mirrors: + if mirror == source_mirror: continue + sys.stdout.write("Updating mirror {0}\n".format(mirror)); sys.stdout.flush() # update this mirror if is_forced: # forcibly update ref remotely (someone already did a force push and hence accepted data loss) - git.push('--force', self.mirrors[mirror], newsha+":"+ref, capture_stderr = suppress_stderr) + git.push('--force', self.mirrors[mirror], newsha+":"+ref) else: # nicely update ref remotely (this avoids data loss due to race conditions) - git.push(self.mirrors[mirror], newsha+":"+ref, capture_stderr = suppress_stderr) + git.push(self.mirrors[mirror], newsha+":"+ref) def update_ref_from_mirror(self, ref, oldsha, newsha, mirror, suppress_stderr = False): '''Update the local version of this to what's currently on the given . and are checked. Then update all the other mirrors.''' - os.chdir(self.local) + self.setup_env() url = self.mirrors[mirror] # first check whether the remote really is at newsha remote_state, code = git.ls_remote(url, ref) @@ -144,7 +156,7 @@ class Repo: assert local_sha in (oldsha, newsha), "Someone lied about the old SHA." # if we are already at newsha locally, we also ran the local hooks, so we do not have to do anything if local_sha == newsha: - return + return "Local repository is already up-to-date." # update local state from local_sha to newsha. if newsha != git_nullsha: # We *could* now fetch the remote ref and immediately update the local one. However, then we would have to @@ -159,8 +171,15 @@ class Repo: # ref does not exist anymore. delete it. assert local_sha != git_nullsha, "Why didn't we bail out earlier if there is nothing to do...?" git.update_ref("-d", ref, local_sha) # this checks that the old value is still local_sha - # update all the mirrors - self.update_mirrors(ref, oldsha, newsha, [mirror], suppress_stderr) + # Now run the post-receive hooks. This will *also* push the changes to all mirrors, as we + # are one of these hooks! + os.putenv("GIT_MIRROR_SOURCE", mirror) # tell ourselves which repo we do *not* have to update + with subprocess.Popen(['/bin/sh', 'hooks/post-receive'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as p: + (stdout, stderr) = p.communicate("{0} {1} {2}\n".format(oldsha, newsha, ref).encode('utf-8')) + stdout = stdout.decode('utf-8') + if p.returncode: + raise Exception("post-receive git hook terminated with non-zero exit code {0}:\n{1}".format(p.returncode, stdout)) + return stdout def find_repo_by_directory(repos, dir): for (name, repo) in repos.items(): @@ -169,8 +188,11 @@ def find_repo_by_directory(repos, dir): return None def load_repos(): + global mail_sender conffile = os.path.join(os.path.dirname(__file__), 'git-mirror.conf') conf = read_config(conffile) + mail_sender = conf['DEFAULT']['mail-sender'] + repos = {} for name, section in conf.items(): if name != 'DEFAULT':