Merge pull request #3 from damien-list/master

[git-mirror.git] / git_mirror.py

diff --git a/git_mirror.py b/git_mirror.py
index 3a65d8594aeb56d74bf07d947cb86866bb751f6f..6b7277592d7cc35ccbb8dd978ae66e8982bc8e94 100644 (file)
--- a/git_mirror.py
+++ b/git_mirror.py
@@ -27,6 +27,18 @@ import hmac, hashlib
 import email.mime.text, email.utils, smtplib
 
 mail_sender = "null@localhost"
 import email.mime.text, email.utils, smtplib
 
 mail_sender = "null@localhost"

+config_file = os.path.join(os.path.dirname(__file__), 'git-mirror.conf')
+
+def Popen_quirky(cmd, **args):
+    '''
+    Runs cmd via subprocess.Popen; and if that fails, puts it into the shell (/bin/sh).
+    It seems that's what executing things in bash does, and even execve.  Also,
+    all so-far released versions of Gitolite get the shebang line wrong.
+    '''
+    try:
+        return subprocess.Popen(cmd, **args)
+    except OSError as e:
+        return subprocess.Popen(['/bin/sh'] + cmd, **args)

 
 class GitCommand:
     def __getattr__(self, name):
 
 class GitCommand:
     def __getattr__(self, name):


@@ -35,12 +47,12 @@ class GitCommand:
                If <check> is true, throw an exception of the process fails with non-zero exit code. Otherwise, do not.
                In any case, return a pair of the captured output and the exit code.'''
             cmd = ["git", name.replace('_', '-')] + list(args)
                If <check> is true, throw an exception of the process fails with non-zero exit code. Otherwise, do not.
                In any case, return a pair of the captured output and the exit code.'''
             cmd = ["git", name.replace('_', '-')] + list(args)

-            with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT if capture_stderr else None) as p:
+            with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT if capture_stderr else sys.stderr) as p:

                 (stdout, stderr) = p.communicate()
                 assert stderr is None
                 code = p.returncode
                 if check and code:
                 (stdout, stderr) = p.communicate()
                 assert stderr is None
                 code = p.returncode
                 if check and code:

-                    raise Exception("Error running {0}: Non-zero exit code".format(cmd))
+                    raise Exception("Error running {}: Non-zero exit code".format(cmd))

             return (stdout.decode('utf-8').strip('\n'), code)
         return call
 
             return (stdout.decode('utf-8').strip('\n'), code)
         return call
 


@@ -53,10 +65,10 @@ def git_is_forced_update(oldsha, newsha):
     assert code in (0, 1)
     return False if code == 0 else True # if oldsha is an ancestor of newsha, then this was a "good" (non-forced) update
 
     assert code in (0, 1)
     return False if code == 0 else True # if oldsha is an ancestor of newsha, then this was a "good" (non-forced) update
 

-def read_config(fname, defSection = 'DEFAULT'):
+def read_config(defSection = 'DEFAULT'):

     '''Reads a config file that may have options outside of any section.'''
     config = configparser.ConfigParser()
     '''Reads a config file that may have options outside of any section.'''
     config = configparser.ConfigParser()

-    with open(fname) as file:
+    with open(config_file) as file:

         stream = itertools.chain(("["+defSection+"]\n",), file)
         config.read_file(stream)
     return config
         stream = itertools.chain(("["+defSection+"]\n",), file)
         config.read_file(stream)
     return config


@@ -83,7 +95,7 @@ class Repo:
         self.name = name
         self.local = conf['local']
         self.owner = conf['owner'] # email address to notify in case of problems
         self.name = name
         self.local = conf['local']
         self.owner = conf['owner'] # email address to notify in case of problems

-        self.hmac_secret = conf['hmac-secret'].encode('utf-8')
+        self.hmac_secret = conf['hmac-secret'].encode('utf-8') if 'hmac-secret' in conf else None

         self.deploy_key = conf['deploy-key'] # the SSH ky used for authenticating against remote hosts
         self.mirrors = {} # maps mirrors to their URLs
         mirror_prefix = 'mirror-'
         self.deploy_key = conf['deploy-key'] # the SSH ky used for authenticating against remote hosts
         self.mirrors = {} # maps mirrors to their URLs
         mirror_prefix = 'mirror-'


@@ -93,9 +105,10 @@ class Repo:
     
     def mail_owner(self, msg):
         global mail_sender
     
     def mail_owner(self, msg):
         global mail_sender

-        send_mail("git-mirror {0}".format(self.name), msg, recipients = [self.owner], sender = mail_sender)
+        send_mail("git-mirror {}".format(self.name), msg, recipients = [self.owner], sender = mail_sender)

 
     def compute_hmac(self, data):
 
     def compute_hmac(self, data):

+        assert self.hmac_secret is not None

         h = hmac.new(self.hmac_secret, digestmod = hashlib.sha1)
         h.update(data)
         return h.hexdigest()
         h = hmac.new(self.hmac_secret, digestmod = hashlib.sha1)
         h.update(data)
         return h.hexdigest()


@@ -112,25 +125,27 @@ class Repo:
         ssh_set_ident = os.path.join(os.path.dirname(__file__), 'ssh-set-ident.sh')
         os.putenv('GIT_SSH', ssh_set_ident)
         ssh_ident = os.path.join(os.path.expanduser('~/.ssh'), self.deploy_key)
         ssh_set_ident = os.path.join(os.path.dirname(__file__), 'ssh-set-ident.sh')
         os.putenv('GIT_SSH', ssh_set_ident)
         ssh_ident = os.path.join(os.path.expanduser('~/.ssh'), self.deploy_key)

-        os.putenv('SSH_IDENT', ssh_ident)
+        os.putenv('GIT_MIRROR_SSH_IDENT', ssh_ident)

     
     

-    def update_mirrors(self, ref, oldsha, newsha, except_mirrors = [], suppress_stderr = False):
+    def update_mirrors(self, ref, oldsha, newsha):

         '''Update the <ref> from <oldsha> to <newsha> on all mirrors. The update must already have happened locally.'''
         assert len(oldsha) == 40 and len(newsha) == 40, "These are not valid SHAs."
         '''Update the <ref> from <oldsha> to <newsha> on all mirrors. The update must already have happened locally.'''
         assert len(oldsha) == 40 and len(newsha) == 40, "These are not valid SHAs."

+        source_mirror = os.getenv("GIT_MIRROR_SOURCE") # in case of a self-call via the hooks, we can skip one of the mirrors

         self.setup_env()
         # check for a forced update
         is_forced = newsha != git_nullsha and oldsha != git_nullsha and git_is_forced_update(oldsha, newsha)
         # tell all the mirrors
         for mirror in self.mirrors:
         self.setup_env()
         # check for a forced update
         is_forced = newsha != git_nullsha and oldsha != git_nullsha and git_is_forced_update(oldsha, newsha)
         # tell all the mirrors
         for mirror in self.mirrors:

-            if mirror in except_mirrors:
+            if mirror == source_mirror:

                 continue
                 continue

+            sys.stdout.write("Updating mirror {}\n".format(mirror)); sys.stdout.flush()

             # update this mirror
             if is_forced:
                 # forcibly update ref remotely (someone already did a force push and hence accepted data loss)
             # update this mirror
             if is_forced:
                 # forcibly update ref remotely (someone already did a force push and hence accepted data loss)

-                git.push('--force', self.mirrors[mirror], newsha+":"+ref, capture_stderr = suppress_stderr)
+                git.push('--force', self.mirrors[mirror], newsha+":"+ref)

             else:
                 # nicely update ref remotely (this avoids data loss due to race conditions)
             else:
                 # nicely update ref remotely (this avoids data loss due to race conditions)

-                git.push(self.mirrors[mirror], newsha+":"+ref, capture_stderr = suppress_stderr)
+                git.push(self.mirrors[mirror], newsha+":"+ref)

     
     def update_ref_from_mirror(self, ref, oldsha, newsha, mirror, suppress_stderr = False):
         '''Update the local version of this <ref> to what's currently on the given <mirror>. <oldsha> and <newsha> are checked. Then update all the other mirrors.'''
     
     def update_ref_from_mirror(self, ref, oldsha, newsha, mirror, suppress_stderr = False):
         '''Update the local version of this <ref> to what's currently on the given <mirror>. <oldsha> and <newsha> are checked. Then update all the other mirrors.'''


@@ -142,19 +157,20 @@ class Repo:
             remote_sha = remote_state.split()[0]
         else:
             remote_sha = git_nullsha
             remote_sha = remote_state.split()[0]
         else:
             remote_sha = git_nullsha

-        assert newsha == remote_sha, "Someone lied about the new SHA, which should be {0}.".format(newsha)
+        assert newsha == remote_sha, "Someone lied about the new SHA, which should be {}.".format(newsha)

         # locally, we have to be at oldsha or newsha (the latter can happen if we already got this update, e.g. if it originated from us)
         local_state, code = git.show_ref(ref, check=False)
         if code == 0:
             local_sha = local_state.split()[0]
         else:
             if len(local_state):
         # locally, we have to be at oldsha or newsha (the latter can happen if we already got this update, e.g. if it originated from us)
         local_state, code = git.show_ref(ref, check=False)
         if code == 0:
             local_sha = local_state.split()[0]
         else:
             if len(local_state):

-                raise Exception("Something went wrong getting the local state of {0}.".format(ref))
+                raise Exception("Something went wrong getting the local state of {}.".format(ref))

             local_sha = git_nullsha
             local_sha = git_nullsha

-        assert local_sha in (oldsha, newsha), "Someone lied about the old SHA."
+        # some sanity checking, but deal gracefully with new branches appearing
+        assert local_sha in (git_nullsha, oldsha, newsha), "Someone lied about the old SHA: Local ({}) is neither old ({}) nor new ({})".format(local_sha, oldsha, newsha)

         # if we are already at newsha locally, we also ran the local hooks, so we do not have to do anything
         if local_sha == newsha:
         # if we are already at newsha locally, we also ran the local hooks, so we do not have to do anything
         if local_sha == newsha:

-            return
+            return "Local repository is already up-to-date."

         # update local state from local_sha to newsha.
         if newsha != git_nullsha:
             # We *could* now fetch the remote ref and immediately update the local one. However, then we would have to
         # update local state from local_sha to newsha.
         if newsha != git_nullsha:
             # We *could* now fetch the remote ref and immediately update the local one. However, then we would have to


@@ -169,8 +185,15 @@ class Repo:
             # ref does not exist anymore. delete it.
             assert local_sha != git_nullsha, "Why didn't we bail out earlier if there is nothing to do...?"
             git.update_ref("-d", ref, local_sha) # this checks that the old value is still local_sha
             # ref does not exist anymore. delete it.
             assert local_sha != git_nullsha, "Why didn't we bail out earlier if there is nothing to do...?"
             git.update_ref("-d", ref, local_sha) # this checks that the old value is still local_sha

-        # update all the mirrors
-        self.update_mirrors(ref, oldsha, newsha, [mirror], suppress_stderr)
+        # Now run the post-receive hooks. This will *also* push the changes to all mirrors, as we
+        # are one of these hooks!
+        os.putenv("GIT_MIRROR_SOURCE", mirror) # tell ourselves which repo we do *not* have to update
+        with Popen_quirky(['hooks/post-receive'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as p:
+            (stdout, stderr) = p.communicate("{} {} {}\n".format(oldsha, newsha, ref).encode('utf-8'))
+            stdout = stdout.decode('utf-8')
+            if p.returncode:
+                raise Exception("post-receive git hook terminated with non-zero exit code {}:\n{}".format(p.returncode, stdout))
+        return stdout

 
 def find_repo_by_directory(repos, dir):
     for (name, repo) in repos.items():
 
 def find_repo_by_directory(repos, dir):
     for (name, repo) in repos.items():


@@ -180,8 +203,7 @@ def find_repo_by_directory(repos, dir):
 
 def load_repos():
     global mail_sender
 
 def load_repos():
     global mail_sender

-    conffile = os.path.join(os.path.dirname(__file__), 'git-mirror.conf')
-    conf = read_config(conffile)
+    conf = read_config()

     mail_sender = conf['DEFAULT']['mail-sender']
     
     repos = {}
     mail_sender = conf['DEFAULT']['mail-sender']
     
     repos = {}