diff options
Diffstat (limited to 'git-p4.py')
| -rwxr-xr-x | git-p4.py | 2428 |
1 files changed, 1468 insertions, 960 deletions
@@ -7,40 +7,62 @@ # 2007 Trolltech ASA # License: MIT <http://www.opensource.org/licenses/mit-license.php> # +# pylint: disable=bad-whitespace +# pylint: disable=broad-except +# pylint: disable=consider-iterating-dictionary +# pylint: disable=disable +# pylint: disable=fixme +# pylint: disable=invalid-name +# pylint: disable=line-too-long +# pylint: disable=missing-docstring +# pylint: disable=no-self-use +# pylint: disable=superfluous-parens +# pylint: disable=too-few-public-methods +# pylint: disable=too-many-arguments +# pylint: disable=too-many-branches +# pylint: disable=too-many-instance-attributes +# pylint: disable=too-many-lines +# pylint: disable=too-many-locals +# pylint: disable=too-many-nested-blocks +# pylint: disable=too-many-statements +# pylint: disable=ungrouped-imports +# pylint: disable=unused-import +# pylint: disable=wrong-import-order +# pylint: disable=wrong-import-position +# + +import struct import sys -if sys.hexversion < 0x02040000: - # The limiter is the subprocess module - sys.stderr.write("git-p4: requires Python 2.4 or later.\n") +if sys.version_info.major < 3 and sys.version_info.minor < 7: + sys.stderr.write("git-p4: requires Python 2.7 or later.\n") sys.exit(1) -import os -import optparse + +import ctypes +import errno +import functools +import glob import marshal -import subprocess -import tempfile -import time +import optparse +import os import platform import re import shutil import stat +import subprocess +import tempfile +import time import zipfile import zlib -import ctypes -import errno +# On python2.7 where raw_input() and input() are both availble, +# we want raw_input's semantics, but aliased to input for python3 +# compatibility +# support basestring in python3 try: - from subprocess import CalledProcessError -except ImportError: - # from python2.7:subprocess.py - # Exception classes used by this module. - class CalledProcessError(Exception): - """This exception is raised when a process run by check_call() returns - a non-zero exit status. The exit status will be stored in the - returncode attribute.""" - def __init__(self, returncode, cmd): - self.returncode = returncode - self.cmd = cmd - def __str__(self): - return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode) + if raw_input and input: + input = raw_input +except: + pass verbose = False @@ -48,22 +70,42 @@ verbose = False defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$' # The block size is reduced automatically if required -defaultBlockSize = 1<<20 +defaultBlockSize = 1 << 20 + +defaultMetadataDecodingStrategy = 'passthrough' if sys.version_info.major == 2 else 'fallback' +defaultFallbackMetadataEncoding = 'cp1252' p4_access_checked = False +re_ko_keywords = re.compile(br'\$(Id|Header)(:[^$\n]+)?\$') +re_k_keywords = re.compile(br'\$(Id|Header|Author|Date|DateTime|Change|File|Revision)(:[^$\n]+)?\$') + + +def format_size_human_readable(num): + """Returns a number of units (typically bytes) formatted as a + human-readable string. + """ + if num < 1024: + return '{:d} B'.format(num) + for unit in ["Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: + num /= 1024.0 + if num < 1024.0: + return "{:3.1f} {}B".format(num, unit) + return "{:.1f} YiB".format(num) + + def p4_build_cmd(cmd): """Build a suitable p4 command line. - This consolidates building and returning a p4 command line into one - location. It means that hooking into the environment, or other configuration - can be done more easily. - """ + This consolidates building and returning a p4 command line into one + location. It means that hooking into the environment, or other + configuration can be done more easily. + """ real_cmd = ["p4"] user = gitConfig("git-p4.user") if len(user) > 0: - real_cmd += ["-u",user] + real_cmd += ["-u", user] password = gitConfig("git-p4.password") if len(password) > 0: @@ -89,10 +131,7 @@ def p4_build_cmd(cmd): # Provide a way to not pass this option by setting git-p4.retries to 0 real_cmd += ["-r", str(retries)] - if isinstance(cmd,basestring): - real_cmd = ' '.join(real_cmd) + ' ' + cmd - else: - real_cmd += cmd + real_cmd += cmd # now check that we can actually talk to the server global p4_access_checked @@ -102,31 +141,31 @@ def p4_build_cmd(cmd): return real_cmd + def git_dir(path): - """ Return TRUE if the given path is a git directory (/path/to/dir/.git). - This won't automatically add ".git" to a directory. - """ + """Return TRUE if the given path is a git directory (/path/to/dir/.git). + This won't automatically add ".git" to a directory. + """ d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip() if not d or len(d) == 0: return None else: return d + def chdir(path, is_client_path=False): - """Do chdir to the given path, and set the PWD environment - variable for use by P4. It does not look at getcwd() output. - Since we're not using the shell, it is necessary to set the - PWD environment variable explicitly. - - Normally, expand the path to force it to be absolute. This - addresses the use of relative path names inside P4 settings, - e.g. P4CONFIG=.p4config. P4 does not simply open the filename - as given; it looks for .p4config using PWD. - - If is_client_path, the path was handed to us directly by p4, - and may be a symbolic link. Do not call os.getcwd() in this - case, because it will cause p4 to think that PWD is not inside - the client path. + """Do chdir to the given path, and set the PWD environment variable for use + by P4. It does not look at getcwd() output. Since we're not using the + shell, it is necessary to set the PWD environment variable explicitly. + + Normally, expand the path to force it to be absolute. This addresses + the use of relative path names inside P4 settings, e.g. + P4CONFIG=.p4config. P4 does not simply open the filename as given; it + looks for .p4config using PWD. + + If is_client_path, the path was handed to us directly by p4, and may be + a symbolic link. Do not call os.getcwd() in this case, because it will + cause p4 to think that PWD is not inside the client path. """ os.chdir(path) @@ -134,6 +173,7 @@ def chdir(path, is_client_path=False): path = os.getcwd() os.environ['PWD'] = path + def calcDiskFree(): """Return free space in bytes on the disk of the given dirname.""" if platform.system() == 'Windows': @@ -144,109 +184,259 @@ def calcDiskFree(): st = os.statvfs(os.getcwd()) return st.f_bavail * st.f_frsize + def die(msg): + """Terminate execution. Make sure that any running child processes have + been wait()ed for before calling this. + """ if verbose: raise Exception(msg) else: sys.stderr.write(msg + "\n") sys.exit(1) -def write_pipe(c, stdin): + +def prompt(prompt_text): + """Prompt the user to choose one of the choices. + + Choices are identified in the prompt_text by square brackets around a + single letter option. + """ + choices = set(m.group(1) for m in re.finditer(r"\[(.)\]", prompt_text)) + while True: + sys.stderr.flush() + sys.stdout.write(prompt_text) + sys.stdout.flush() + response = sys.stdin.readline().strip().lower() + if not response: + continue + response = response[0] + if response in choices: + return response + + +# We need different encoding/decoding strategies for text data being passed +# around in pipes depending on python version +if bytes is not str: + # For python3, always encode and decode as appropriate + def decode_text_stream(s): + return s.decode() if isinstance(s, bytes) else s + + def encode_text_stream(s): + return s.encode() if isinstance(s, str) else s +else: + # For python2.7, pass read strings as-is, but also allow writing unicode + def decode_text_stream(s): + return s + + def encode_text_stream(s): + return s.encode('utf_8') if isinstance(s, unicode) else s + + +class MetadataDecodingException(Exception): + def __init__(self, input_string): + self.input_string = input_string + + def __str__(self): + return """Decoding perforce metadata failed! +The failing string was: +--- +{} +--- +Consider setting the git-p4.metadataDecodingStrategy config option to +'fallback', to allow metadata to be decoded using a fallback encoding, +defaulting to cp1252.""".format(self.input_string) + + +encoding_fallback_warning_issued = False +encoding_escape_warning_issued = False +def metadata_stream_to_writable_bytes(s): + encodingStrategy = gitConfig('git-p4.metadataDecodingStrategy') or defaultMetadataDecodingStrategy + fallbackEncoding = gitConfig('git-p4.metadataFallbackEncoding') or defaultFallbackMetadataEncoding + if not isinstance(s, bytes): + return s.encode('utf_8') + if encodingStrategy == 'passthrough': + return s + try: + s.decode('utf_8') + return s + except UnicodeDecodeError: + if encodingStrategy == 'fallback' and fallbackEncoding: + global encoding_fallback_warning_issued + global encoding_escape_warning_issued + try: + if not encoding_fallback_warning_issued: + print("\nCould not decode value as utf-8; using configured fallback encoding %s: %s" % (fallbackEncoding, s)) + print("\n(this warning is only displayed once during an import)") + encoding_fallback_warning_issued = True + return s.decode(fallbackEncoding).encode('utf_8') + except Exception as exc: + if not encoding_escape_warning_issued: + print("\nCould not decode value with configured fallback encoding %s; escaping bytes over 127: %s" % (fallbackEncoding, s)) + print("\n(this warning is only displayed once during an import)") + encoding_escape_warning_issued = True + escaped_bytes = b'' + # bytes and strings work very differently in python2 vs python3... + if str is bytes: + for byte in s: + byte_number = struct.unpack('>B', byte)[0] + if byte_number > 127: + escaped_bytes += b'%' + escaped_bytes += hex(byte_number)[2:].upper() + else: + escaped_bytes += byte + else: + for byte_number in s: + if byte_number > 127: + escaped_bytes += b'%' + escaped_bytes += hex(byte_number).upper().encode()[2:] + else: + escaped_bytes += bytes([byte_number]) + return escaped_bytes + + raise MetadataDecodingException(s) + + +def decode_path(path): + """Decode a given string (bytes or otherwise) using configured path + encoding options. + """ + + encoding = gitConfig('git-p4.pathEncoding') or 'utf_8' + if bytes is not str: + return path.decode(encoding, errors='replace') if isinstance(path, bytes) else path + else: + try: + path.decode('ascii') + except: + path = path.decode(encoding, errors='replace') + if verbose: + print('Path with non-ASCII characters detected. Used {} to decode: {}'.format(encoding, path)) + return path + + +def run_git_hook(cmd, param=[]): + """Execute a hook if the hook exists.""" + args = ['git', 'hook', 'run', '--ignore-missing', cmd] + if param: + args.append("--") + for p in param: + args.append(p) + return subprocess.call(args) == 0 + + +def write_pipe(c, stdin, *k, **kw): if verbose: - sys.stderr.write('Writing pipe: %s\n' % str(c)) + sys.stderr.write('Writing pipe: {}\n'.format(' '.join(c))) - expand = isinstance(c,basestring) - p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand) + p = subprocess.Popen(c, stdin=subprocess.PIPE, *k, **kw) pipe = p.stdin val = pipe.write(stdin) pipe.close() if p.wait(): - die('Command failed: %s' % str(c)) + die('Command failed: {}'.format(' '.join(c))) return val -def p4_write_pipe(c, stdin): + +def p4_write_pipe(c, stdin, *k, **kw): real_cmd = p4_build_cmd(c) - return write_pipe(real_cmd, stdin) + if bytes is not str and isinstance(stdin, str): + stdin = encode_text_stream(stdin) + return write_pipe(real_cmd, stdin, *k, **kw) -def read_pipe_full(c): - """ Read output from command. Returns a tuple - of the return status, stdout text and stderr - text. - """ + +def read_pipe_full(c, *k, **kw): + """Read output from command. Returns a tuple of the return status, stdout + text and stderr text. + """ if verbose: - sys.stderr.write('Reading pipe: %s\n' % str(c)) + sys.stderr.write('Reading pipe: {}\n'.format(' '.join(c))) - expand = isinstance(c,basestring) - p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand) - (out, err) = p.communicate() - return (p.returncode, out, err) + p = subprocess.Popen( + c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, *k, **kw) + out, err = p.communicate() + return (p.returncode, out, decode_text_stream(err)) -def read_pipe(c, ignore_error=False): - """ Read output from command. Returns the output text on - success. On failure, terminates execution, unless - ignore_error is True, when it returns an empty string. - """ - (retcode, out, err) = read_pipe_full(c) + +def read_pipe(c, ignore_error=False, raw=False, *k, **kw): + """Read output from command. Returns the output text on success. On + failure, terminates execution, unless ignore_error is True, when it + returns an empty string. + + If raw is True, do not attempt to decode output text. + """ + retcode, out, err = read_pipe_full(c, *k, **kw) if retcode != 0: if ignore_error: out = "" else: - die('Command failed: %s\nError: %s' % (str(c), err)) + die('Command failed: {}\nError: {}'.format(' '.join(c), err)) + if not raw: + out = decode_text_stream(out) return out -def read_pipe_text(c): - """ Read output from a command with trailing whitespace stripped. - On error, returns None. - """ - (retcode, out, err) = read_pipe_full(c) + +def read_pipe_text(c, *k, **kw): + """Read output from a command with trailing whitespace stripped. On error, + returns None. + """ + retcode, out, err = read_pipe_full(c, *k, **kw) if retcode != 0: return None else: - return out.rstrip() + return decode_text_stream(out).rstrip() -def p4_read_pipe(c, ignore_error=False): + +def p4_read_pipe(c, ignore_error=False, raw=False, *k, **kw): real_cmd = p4_build_cmd(c) - return read_pipe(real_cmd, ignore_error) + return read_pipe(real_cmd, ignore_error, raw=raw, *k, **kw) + -def read_pipe_lines(c): +def read_pipe_lines(c, raw=False, *k, **kw): if verbose: - sys.stderr.write('Reading pipe: %s\n' % str(c)) + sys.stderr.write('Reading pipe: {}\n'.format(' '.join(c))) - expand = isinstance(c, basestring) - p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) + p = subprocess.Popen(c, stdout=subprocess.PIPE, *k, **kw) pipe = p.stdout - val = pipe.readlines() + lines = pipe.readlines() + if not raw: + lines = [decode_text_stream(line) for line in lines] if pipe.close() or p.wait(): - die('Command failed: %s' % str(c)) + die('Command failed: {}'.format(' '.join(c))) + return lines - return val -def p4_read_pipe_lines(c): - """Specifically invoke p4 on the command supplied. """ +def p4_read_pipe_lines(c, *k, **kw): + """Specifically invoke p4 on the command supplied.""" real_cmd = p4_build_cmd(c) - return read_pipe_lines(real_cmd) + return read_pipe_lines(real_cmd, *k, **kw) + def p4_has_command(cmd): - """Ask p4 for help on this command. If it returns an error, the - command does not exist in this version of p4.""" + """Ask p4 for help on this command. If it returns an error, the command + does not exist in this version of p4. + """ real_cmd = p4_build_cmd(["help", cmd]) p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p.communicate() return p.returncode == 0 + def p4_has_move_command(): - """See if the move command exists, that it supports -k, and that - it has not been administratively disabled. The arguments - must be correct, but the filenames do not have to exist. Use - ones with wildcards so even if they exist, it will fail.""" + """See if the move command exists, that it supports -k, and that it has not + been administratively disabled. The arguments must be correct, but the + filenames do not have to exist. Use ones with wildcards so even if they + exist, it will fail. + """ if not p4_has_command("move"): return False cmd = p4_build_cmd(["move", "-k", "@from", "@to"]) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (out, err) = p.communicate() + out, err = p.communicate() + err = decode_text_stream(err) # return code will be 1 in either case if err.find("Invalid option") >= 0: return False @@ -255,30 +445,33 @@ def p4_has_move_command(): # assume it failed because @... was invalid changelist return True -def system(cmd, ignore_error=False): - expand = isinstance(cmd,basestring) + +def system(cmd, ignore_error=False, *k, **kw): if verbose: - sys.stderr.write("executing %s\n" % str(cmd)) - retcode = subprocess.call(cmd, shell=expand) + sys.stderr.write("executing {}\n".format( + ' '.join(cmd) if isinstance(cmd, list) else cmd)) + retcode = subprocess.call(cmd, *k, **kw) if retcode and not ignore_error: - raise CalledProcessError(retcode, cmd) + raise subprocess.CalledProcessError(retcode, cmd) return retcode -def p4_system(cmd): - """Specifically invoke p4 as the system command. """ + +def p4_system(cmd, *k, **kw): + """Specifically invoke p4 as the system command.""" real_cmd = p4_build_cmd(cmd) - expand = isinstance(real_cmd, basestring) - retcode = subprocess.call(real_cmd, shell=expand) + retcode = subprocess.call(real_cmd, *k, **kw) if retcode: - raise CalledProcessError(retcode, real_cmd) + raise subprocess.CalledProcessError(retcode, real_cmd) + def die_bad_access(s): die("failure accessing depot: {0}".format(s.rstrip())) + def p4_check_access(min_expiration=1): - """ Check if we can access Perforce - account still logged in - """ + """Check if we can access Perforce - account still logged in.""" + results = p4CmdList(["login", "-s"]) if len(results) == 0: @@ -316,65 +509,83 @@ def p4_check_access(min_expiration=1): die_bad_access("p4 error: {0}".format(data)) else: die_bad_access("unknown error") + elif code == "info": + return else: die_bad_access("unknown error code {0}".format(code)) + _p4_version_string = None + + def p4_version_string(): - """Read the version string, showing just the last line, which - hopefully is the interesting version bit. + """Read the version string, showing just the last line, which hopefully is + the interesting version bit. $ p4 -V Perforce - The Fast Software Configuration Management System. Copyright 1995-2011 Perforce Software. All rights reserved. Rev. P4/NTX86/2011.1/393975 (2011/12/16). - """ + """ global _p4_version_string if not _p4_version_string: a = p4_read_pipe_lines(["-V"]) _p4_version_string = a[-1].rstrip() return _p4_version_string + def p4_integrate(src, dest): p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)]) + def p4_sync(f, *options): p4_system(["sync"] + list(options) + [wildcard_encode(f)]) + def p4_add(f): - # forcibly add file names with wildcards + """Forcibly add file names with wildcards.""" if wildcard_present(f): p4_system(["add", "-f", f]) else: p4_system(["add", f]) + def p4_delete(f): p4_system(["delete", wildcard_encode(f)]) + def p4_edit(f, *options): p4_system(["edit"] + list(options) + [wildcard_encode(f)]) + def p4_revert(f): p4_system(["revert", wildcard_encode(f)]) + def p4_reopen(type, f): p4_system(["reopen", "-t", type, wildcard_encode(f)]) + def p4_reopen_in_change(changelist, files): cmd = ["reopen", "-c", str(changelist)] + files p4_system(cmd) + def p4_move(src, dest): p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)]) + def p4_last_change(): results = p4CmdList(["changes", "-m", "1"], skip_info=True) return int(results[0]['change']) + def p4_describe(change, shelved=False): - """Make sure it returns a valid result by checking for - the presence of field "time". Return a dict of the - results.""" + """Make sure it returns a valid result by checking for the presence of + field "time". + + Return a dict of the results. + """ cmd = ["describe", "-s"] if shelved: @@ -399,12 +610,11 @@ def p4_describe(change, shelved=False): return d -# -# Canonicalize the p4 type and return a tuple of the -# base type, plus any modifiers. See "p4 help filetypes" -# for a list and explanation. -# + def split_p4_type(p4type): + """Canonicalize the p4 type and return a tuple of the base type, plus any + modifiers. See "p4 help filetypes" for a list and explanation. + """ p4_filetypes_historical = { "ctempobj": "binary+Sw", @@ -434,51 +644,46 @@ def split_p4_type(p4type): mods = s[1] return (base, mods) -# -# return the raw p4 type of a file (text, text+ko, etc) -# + def p4_type(f): + """Return the raw p4 type of a file (text, text+ko, etc).""" + results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)]) return results[0]['headType'] -# -# Given a type base and modifier, return a regexp matching -# the keywords that can be expanded in the file -# + def p4_keywords_regexp_for_type(base, type_mods): + """Given a type base and modifier, return a regexp matching the keywords + that can be expanded in the file. + """ + if base in ("text", "unicode", "binary"): - kwords = None if "ko" in type_mods: - kwords = 'Id|Header' + return re_ko_keywords elif "k" in type_mods: - kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision' + return re_k_keywords else: return None - pattern = r""" - \$ # Starts with a dollar, followed by... - (%s) # one of the keywords, followed by... - (:[^$\n]+)? # possibly an old expansion, followed by... - \$ # another dollar - """ % kwords - return pattern else: return None -# -# Given a file, return a regexp matching the possible -# RCS keywords that will be expanded, or None for files -# with kw expansion turned off. -# + def p4_keywords_regexp_for_file(file): + """Given a file, return a regexp matching the possible RCS keywords that + will be expanded, or None for files with kw expansion turned off. + """ + if not os.path.exists(file): return None else: - (type_base, type_mods) = split_p4_type(p4_type(file)) + type_base, type_mods = split_p4_type(p4_type(file)) return p4_keywords_regexp_for_type(type_base, type_mods) + def setP4ExecBit(file, mode): - # Reopens an already open file and changes the execute bit to match - # the execute bit setting in the passed in mode. + """Reopens an already open file and changes the execute bit to match the + execute bit setting in the passed in mode. + """ p4Type = "+x" @@ -491,8 +696,9 @@ def setP4ExecBit(file, mode): p4_reopen(p4Type, file) + def getP4OpenedType(file): - # Returns the perforce file type for the given file. + """Returns the perforce file type for the given file.""" result = p4_read_pipe(["opened", wildcard_encode(file)]) match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result) @@ -501,10 +707,12 @@ def getP4OpenedType(file): else: die("Could not determine file type for %s (result: '%s')" % (file, result)) -# Return the set of all p4 labels + def getP4Labels(depotPaths): + """Return the set of all p4 labels.""" + labels = set() - if isinstance(depotPaths,basestring): + if not isinstance(depotPaths, list): depotPaths = [depotPaths] for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]): @@ -513,41 +721,45 @@ def getP4Labels(depotPaths): return labels -# Return the set of all git tags + def getGitTags(): + """Return the set of all git tags.""" + gitTags = set() for line in read_pipe_lines(["git", "tag"]): tag = line.strip() gitTags.add(tag) return gitTags -def diffTreePattern(): - # This is a simple generator for the diff tree regex pattern. This could be - # a class variable if this and parseDiffTreeEntry were a part of a class. - pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)') - while True: - yield pattern + +_diff_tree_pattern = None + def parseDiffTreeEntry(entry): """Parses a single diff tree entry into its component elements. - See git-diff-tree(1) manpage for details about the format of the diff - output. This method returns a dictionary with the following elements: - - src_mode - The mode of the source file - dst_mode - The mode of the destination file - src_sha1 - The sha1 for the source file - dst_sha1 - The sha1 fr the destination file - status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc) - status_score - The score for the status (applicable for 'C' and 'R' - statuses). This is None if there is no score. - src - The path for the source file. - dst - The path for the destination file. This is only present for - copy or renames. If it is not present, this is None. + See git-diff-tree(1) manpage for details about the format of the diff + output. This method returns a dictionary with the following elements: + + src_mode - The mode of the source file + dst_mode - The mode of the destination file + src_sha1 - The sha1 for the source file + dst_sha1 - The sha1 fr the destination file + status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc) + status_score - The score for the status (applicable for 'C' and 'R' + statuses). This is None if there is no score. + src - The path for the source file. + dst - The path for the destination file. This is only present for + copy or renames. If it is not present, this is None. + + If the pattern is not matched, None is returned. + """ - If the pattern is not matched, None is returned.""" + global _diff_tree_pattern + if not _diff_tree_pattern: + _diff_tree_pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)') - match = diffTreePattern().next().match(entry) + match = _diff_tree_pattern.match(entry) if match: return { 'src_mode': match.group(1), @@ -561,46 +773,97 @@ def parseDiffTreeEntry(entry): } return None + def isModeExec(mode): - # Returns True if the given git mode represents an executable file, - # otherwise False. + """Returns True if the given git mode represents an executable file, + otherwise False. + """ return mode[-3:] == "755" + class P4Exception(Exception): - """ Base class for exceptions from the p4 client """ + """Base class for exceptions from the p4 client.""" + def __init__(self, exit_code): self.p4ExitCode = exit_code + class P4ServerException(P4Exception): - """ Base class for exceptions where we get some kind of marshalled up result from the server """ + """Base class for exceptions where we get some kind of marshalled up result + from the server. + """ + def __init__(self, exit_code, p4_result): super(P4ServerException, self).__init__(exit_code) self.p4_result = p4_result self.code = p4_result[0]['code'] self.data = p4_result[0]['data'] + class P4RequestSizeException(P4ServerException): - """ One of the maxresults or maxscanrows errors """ + """One of the maxresults or maxscanrows errors.""" + def __init__(self, exit_code, p4_result, limit): super(P4RequestSizeException, self).__init__(exit_code, p4_result) self.limit = limit + +class P4CommandException(P4Exception): + """Something went wrong calling p4 which means we have to give up.""" + + def __init__(self, msg): + self.msg = msg + + def __str__(self): + return self.msg + + def isModeExecChanged(src_mode, dst_mode): return isModeExec(src_mode) != isModeExec(dst_mode) -def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False, - errors_as_exceptions=False): - if isinstance(cmd,basestring): - cmd = "-G " + cmd - expand = True - else: - cmd = ["-G"] + cmd - expand = False +def p4KeysContainingNonUtf8Chars(): + """Returns all keys which may contain non UTF-8 encoded strings + for which a fallback strategy has to be applied. + """ + return ['desc', 'client', 'FullName'] + + +def p4KeysContainingBinaryData(): + """Returns all keys which may contain arbitrary binary data + """ + return ['data'] + - cmd = p4_build_cmd(cmd) +def p4KeyContainsFilePaths(key): + """Returns True if the key contains file paths. These are handled by decode_path(). + Otherwise False. + """ + return key.startswith('depotFile') or key in ['path', 'clientFile'] + + +def p4KeyWhichCanBeDirectlyDecoded(key): + """Returns True if the key can be directly decoded as UTF-8 string + Otherwise False. + + Keys which can not be encoded directly: + - `data` which may contain arbitrary binary data + - `desc` or `client` or `FullName` which may contain non-UTF8 encoded text + - `depotFile[0-9]*`, `path`, or `clientFile` which may contain non-UTF8 encoded text, handled by decode_path() + """ + if key in p4KeysContainingNonUtf8Chars() or \ + key in p4KeysContainingBinaryData() or \ + p4KeyContainsFilePaths(key): + return False + return True + + +def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False, + errors_as_exceptions=False, *k, **kw): + + cmd = p4_build_cmd(["-G"] + cmd) if verbose: - sys.stderr.write("Opening pipe: %s\n" % str(cmd)) + sys.stderr.write("Opening pipe: {}\n".format(' '.join(cmd))) # Use a temporary file to avoid deadlocks without # subprocess.communicate(), which would put another copy @@ -608,26 +871,41 @@ def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False, stdin_file = None if stdin is not None: stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode) - if isinstance(stdin,basestring): + if not isinstance(stdin, list): stdin_file.write(stdin) else: for i in stdin: - stdin_file.write(i + '\n') + stdin_file.write(encode_text_stream(i)) + stdin_file.write(b'\n') stdin_file.flush() stdin_file.seek(0) - p4 = subprocess.Popen(cmd, - shell=expand, - stdin=stdin_file, - stdout=subprocess.PIPE) + p4 = subprocess.Popen( + cmd, stdin=stdin_file, stdout=subprocess.PIPE, *k, **kw) result = [] try: while True: entry = marshal.load(p4.stdout) + + if bytes is not str: + # Decode unmarshalled dict to use str keys and values. Special cases are handled below. + decoded_entry = {} + for key, value in entry.items(): + key = key.decode() + if isinstance(value, bytes) and p4KeyWhichCanBeDirectlyDecoded(key): + value = value.decode() + decoded_entry[key] = value + # Parse out data if it's an error response + if decoded_entry.get('code') == 'error' and 'data' in decoded_entry: + decoded_entry['data'] = decoded_entry['data'].decode() + entry = decoded_entry if skip_info: if 'code' in entry and entry['code'] == 'info': continue + for key in p4KeysContainingNonUtf8Chars(): + if key in entry: + entry[key] = metadata_stream_to_writable_bytes(entry[key]) if cb is not None: cb(entry) else: @@ -658,12 +936,14 @@ def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False, return result -def p4Cmd(cmd): - list = p4CmdList(cmd) + +def p4Cmd(cmd, *k, **kw): + list = p4CmdList(cmd, *k, **kw) result = {} for entry in list: result.update(entry) - return result; + return result + def p4Where(depotPath): if not depotPath.endswith("/"): @@ -675,7 +955,8 @@ def p4Where(depotPath): if "depotFile" in entry: # Search for the base client side depot path, as long as it starts with the branch's P4 path. # The base path always ends with "/...". - if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...": + entry_path = decode_path(entry['depotFile']) + if entry_path.find(depotPath) == 0 and entry_path[-4:] == "/...": output = entry break elif "data" in entry: @@ -684,64 +965,70 @@ def p4Where(depotPath): if data[:space] == depotPath: output = entry break - if output == None: + if output is None: return "" if output["code"] == "error": return "" clientPath = "" if "path" in output: - clientPath = output.get("path") + clientPath = decode_path(output['path']) elif "data" in output: data = output.get("data") - lastSpace = data.rfind(" ") - clientPath = data[lastSpace + 1:] + lastSpace = data.rfind(b" ") + clientPath = decode_path(data[lastSpace + 1:]) if clientPath.endswith("..."): clientPath = clientPath[:-3] return clientPath + def currentGitBranch(): return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"]) + def isValidGitDir(path): - return git_dir(path) != None + return git_dir(path) is not None + def parseRevision(ref): - return read_pipe("git rev-parse %s" % ref).strip() + return read_pipe(["git", "rev-parse", ref]).strip() + def branchExists(ref): rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref], ignore_error=True) return len(rev) > 0 + def extractLogMessageFromGitCommit(commit): logMessage = "" - ## fixme: title is first line of commit, not 1st paragraph. + # fixme: title is first line of commit, not 1st paragraph. foundTitle = False - for log in read_pipe_lines("git cat-file commit %s" % commit): - if not foundTitle: - if len(log) == 1: - foundTitle = True - continue + for log in read_pipe_lines(["git", "cat-file", "commit", commit]): + if not foundTitle: + if len(log) == 1: + foundTitle = True + continue - logMessage += log + logMessage += log return logMessage + def extractSettingsGitLog(log): values = {} for line in log.split("\n"): line = line.strip() - m = re.search (r"^ *\[git-p4: (.*)\]$", line) + m = re.search(r"^ *\[git-p4: (.*)\]$", line) if not m: continue - assignments = m.group(1).split (':') + assignments = m.group(1).split(':') for a in assignments: - vals = a.split ('=') + vals = a.split('=') key = vals[0].strip() - val = ('='.join (vals[1:])).strip() - if val.endswith ('\"') and val.startswith('"'): + val = ('='.join(vals[1:])).strip() + if val.endswith('\"') and val.startswith('"'): val = val[1:-1] values[key] = val @@ -753,41 +1040,49 @@ def extractSettingsGitLog(log): values['depot-paths'] = paths.split(',') return values + def gitBranchExists(branch): proc = subprocess.Popen(["git", "rev-parse", branch], - stderr=subprocess.PIPE, stdout=subprocess.PIPE); - return proc.wait() == 0; + stderr=subprocess.PIPE, stdout=subprocess.PIPE) + return proc.wait() == 0 + def gitUpdateRef(ref, newvalue): subprocess.check_call(["git", "update-ref", ref, newvalue]) + def gitDeleteRef(ref): subprocess.check_call(["git", "update-ref", "-d", ref]) + _gitConfig = {} + def gitConfig(key, typeSpecifier=None): - if not _gitConfig.has_key(key): - cmd = [ "git", "config" ] + if key not in _gitConfig: + cmd = ["git", "config"] if typeSpecifier: - cmd += [ typeSpecifier ] - cmd += [ key ] + cmd += [typeSpecifier] + cmd += [key] s = read_pipe(cmd, ignore_error=True) _gitConfig[key] = s.strip() return _gitConfig[key] + def gitConfigBool(key): """Return a bool, using git config --bool. It is True only if the variable is set to true, and False if set to false or not present - in the config.""" + in the config. + """ - if not _gitConfig.has_key(key): + if key not in _gitConfig: _gitConfig[key] = gitConfig(key, '--bool') == "true" return _gitConfig[key] + def gitConfigInt(key): - if not _gitConfig.has_key(key): - cmd = [ "git", "config", "--int", key ] + if key not in _gitConfig: + cmd = ["git", "config", "--int", key] s = read_pipe(cmd, ignore_error=True) v = s.strip() try: @@ -796,28 +1091,60 @@ def gitConfigInt(key): _gitConfig[key] = None return _gitConfig[key] + def gitConfigList(key): - if not _gitConfig.has_key(key): + if key not in _gitConfig: s = read_pipe(["git", "config", "--get-all", key], ignore_error=True) _gitConfig[key] = s.strip().splitlines() if _gitConfig[key] == ['']: _gitConfig[key] = [] return _gitConfig[key] +def fullP4Ref(incomingRef, importIntoRemotes=True): + """Standardize a given provided p4 ref value to a full git ref: + refs/foo/bar/branch -> use it exactly + p4/branch -> prepend refs/remotes/ or refs/heads/ + branch -> prepend refs/remotes/p4/ or refs/heads/p4/""" + if incomingRef.startswith("refs/"): + return incomingRef + if importIntoRemotes: + prepend = "refs/remotes/" + else: + prepend = "refs/heads/" + if not incomingRef.startswith("p4/"): + prepend += "p4/" + return prepend + incomingRef + +def shortP4Ref(incomingRef, importIntoRemotes=True): + """Standardize to a "short ref" if possible: + refs/foo/bar/branch -> ignore + refs/remotes/p4/branch or refs/heads/p4/branch -> shorten + p4/branch -> shorten""" + if importIntoRemotes: + longprefix = "refs/remotes/p4/" + else: + longprefix = "refs/heads/p4/" + if incomingRef.startswith(longprefix): + return incomingRef[len(longprefix):] + if incomingRef.startswith("p4/"): + return incomingRef[3:] + return incomingRef + def p4BranchesInGit(branchesAreInRemotes=True): """Find all the branches whose names start with "p4/", looking in remotes or heads as specified by the argument. Return a dictionary of { branch: revision } for each one found. The branch names are the short names, without any - "p4/" prefix.""" + "p4/" prefix. + """ branches = {} - cmdline = "git rev-parse --symbolic " + cmdline = ["git", "rev-parse", "--symbolic"] if branchesAreInRemotes: - cmdline += "--remotes" + cmdline.append("--remotes") else: - cmdline += "--branches" + cmdline.append("--branches") for line in read_pipe_lines(cmdline): line = line.strip() @@ -836,18 +1163,21 @@ def p4BranchesInGit(branchesAreInRemotes=True): return branches + def branch_exists(branch): """Make sure that the given ref name really exists.""" - cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ] + cmd = ["git", "rev-parse", "--symbolic", "--verify", branch] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _ = p.communicate() + out = decode_text_stream(out) if p.returncode: return False # expect exactly one line of output: the branch name return out.rstrip() == branch -def findUpstreamBranchPoint(head = "HEAD"): + +def findUpstreamBranchPoint(head="HEAD"): branches = p4BranchesInGit() # map from depot-path to branch name branchByDepotPath = {} @@ -855,9 +1185,13 @@ def findUpstreamBranchPoint(head = "HEAD"): tip = branches[branch] log = extractLogMessageFromGitCommit(tip) settings = extractSettingsGitLog(log) - if settings.has_key("depot-paths"): + if "depot-paths" in settings: + git_branch = "remotes/p4/" + branch paths = ",".join(settings["depot-paths"]) - branchByDepotPath[paths] = "remotes/p4/" + branch + branchByDepotPath[paths] = git_branch + if "change" in settings: + paths = paths + ";" + settings["change"] + branchByDepotPath[paths] = git_branch settings = None parent = 0 @@ -865,23 +1199,28 @@ def findUpstreamBranchPoint(head = "HEAD"): commit = head + "~%s" % parent log = extractLogMessageFromGitCommit(commit) settings = extractSettingsGitLog(log) - if settings.has_key("depot-paths"): + if "depot-paths" in settings: paths = ",".join(settings["depot-paths"]) - if branchByDepotPath.has_key(paths): + if "change" in settings: + expaths = paths + ";" + settings["change"] + if expaths in branchByDepotPath: + return [branchByDepotPath[expaths], settings] + if paths in branchByDepotPath: return [branchByDepotPath[paths], settings] parent = parent + 1 return ["", settings] -def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True): + +def createOrUpdateBranchesFromOrigin(localRefPrefix="refs/remotes/p4/", silent=True): if not silent: - print ("Creating/updating branch(es) in %s based on origin branch(es)" + print("Creating/updating branch(es) in %s based on origin branch(es)" % localRefPrefix) originPrefix = "origin/p4/" - for line in read_pipe_lines("git rev-parse --symbolic --remotes"): + for line in read_pipe_lines(["git", "rev-parse", "--symbolic", "--remotes"]): line = line.strip() if (not line.startswith(originPrefix)) or line.endswith("HEAD"): continue @@ -891,38 +1230,38 @@ def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent originHead = line original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) - if (not original.has_key('depot-paths') - or not original.has_key('change')): + if 'depot-paths' not in original or 'change' not in original: continue update = False if not gitBranchExists(remoteHead): if verbose: - print "creating %s" % remoteHead + print("creating %s" % remoteHead) update = True else: settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) - if settings.has_key('change') > 0: + if 'change' in settings: if settings['depot-paths'] == original['depot-paths']: originP4Change = int(original['change']) p4Change = int(settings['change']) if originP4Change > p4Change: - print ("%s (%s) is newer than %s (%s). " + print("%s (%s) is newer than %s (%s). " "Updating p4 branch from origin." % (originHead, originP4Change, remoteHead, p4Change)) update = True else: - print ("Ignoring: %s was imported from %s while " + print("Ignoring: %s was imported from %s while " "%s was imported from %s" % (originHead, ','.join(original['depot-paths']), remoteHead, ','.join(settings['depot-paths']))) if update: - system("git update-ref %s %s" % (remoteHead, originHead)) + system(["git", "update-ref", remoteHead, originHead]) + def originP4BranchesExist(): - return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master") + return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master") def p4ParseNumericChangeRange(parts): @@ -934,12 +1273,14 @@ def p4ParseNumericChangeRange(parts): return (changeStart, changeEnd) + def chooseBlockSize(blockSize): if blockSize: return blockSize else: return defaultBlockSize + def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize): assert depotPaths @@ -957,7 +1298,7 @@ def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize): parts = changeRange.split(',') assert len(parts) == 2 try: - (changeStart, changeEnd) = p4ParseNumericChangeRange(parts) + changeStart, changeEnd = p4ParseNumericChangeRange(parts) block_size = chooseBlockSize(requestedBlockSize) except ValueError: changeStart = parts[0][1:] @@ -995,14 +1336,15 @@ def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize): else: block_size = max(2, block_size // 2) - if verbose: print("block size error, retrying with block size {0}".format(block_size)) + if verbose: + print("block size error, retrying with block size {0}".format(block_size)) continue except P4Exception as e: die('Error retrieving changes description ({0})'.format(e.p4ExitCode)) # Insert changes in chronological order for entry in reversed(result): - if not entry.has_key('change'): + if 'change' not in entry: continue changes.add(int(entry['change'])) @@ -1017,23 +1359,27 @@ def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize): changes = sorted(changes) return changes + def p4PathStartsWith(path, prefix): - # This method tries to remedy a potential mixed-case issue: - # - # If UserA adds //depot/DirA/file1 - # and UserB adds //depot/dira/file2 - # - # we may or may not have a problem. If you have core.ignorecase=true, - # we treat DirA and dira as the same directory + """This method tries to remedy a potential mixed-case issue: + + If UserA adds //depot/DirA/file1 + and UserB adds //depot/dira/file2 + + we may or may not have a problem. If you have core.ignorecase=true, + we treat DirA and dira as the same directory. + """ if gitConfigBool("core.ignorecase"): return path.lower().startswith(prefix.lower()) return path.startswith(prefix) + def getClientSpec(): """Look at the p4 client spec, create a View() object that contains - all the mappings, and return it.""" + all the mappings, and return it. + """ - specList = p4CmdList("client -o") + specList = p4CmdList(["client", "-o"]) if len(specList) != 1: die('Output from "client -o" is %d lines, expecting 1' % len(specList)) @@ -1045,7 +1391,7 @@ def getClientSpec(): client_name = entry["Client"] # just the keys that start with "View" - view_keys = [ k for k in entry.keys() if k.startswith("View") ] + view_keys = [k for k in entry.keys() if k.startswith("View")] # hold this new View view = View(client_name) @@ -1059,10 +1405,11 @@ def getClientSpec(): return view + def getClientRoot(): """Grab the client directory.""" - output = p4CmdList("client -o") + output = p4CmdList(["client", "-o"]) if len(output) != 1: die('Output from "client -o" is %d lines, expecting 1' % len(output)) @@ -1072,12 +1419,15 @@ def getClientRoot(): return entry["Root"] -# -# P4 wildcards are not allowed in filenames. P4 complains -# if you simply add them, but you can force it with "-f", in -# which case it translates them into %xx encoding internally. -# + def wildcard_decode(path): + """Decode P4 wildcards into %xx encoding + + P4 wildcards are not allowed in filenames. P4 complains if you simply + add them, but you can force it with "-f", in which case it translates + them into %xx encoding internally. + """ + # Search for and fix just these four characters. Do % last so # that fixing it does not inadvertently create new %-escapes. # Cannot have * in a filename in windows; untested as to @@ -1089,7 +1439,10 @@ def wildcard_decode(path): .replace("%25", "%") return path + def wildcard_encode(path): + """Encode %xx coded wildcards into P4 coding.""" + # do % first to avoid double-encoding the %s introduced here path = path.replace("%", "%25") \ .replace("*", "%2A") \ @@ -1097,10 +1450,12 @@ def wildcard_encode(path): .replace("@", "%40") return path + def wildcard_present(path): m = re.search("[*#@%]", path) return m is not None + class LargeFileSystem(object): """Base class for large file system support.""" @@ -1109,17 +1464,19 @@ class LargeFileSystem(object): self.writeToGitStream = writeToGitStream def generatePointer(self, cloneDestination, contentFile): - """Return the content of a pointer file that is stored in Git instead of - the actual content.""" + """Return the content of a pointer file that is stored in Git instead + of the actual content. + """ assert False, "Method 'generatePointer' required in " + self.__class__.__name__ def pushFile(self, localLargeFile): """Push the actual content which is not stored in the Git repository to - a server.""" + a server. + """ assert False, "Method 'pushFile' required in " + self.__class__.__name__ def hasLargeFileExtension(self, relPath): - return reduce( + return functools.reduce( lambda a, b: a or b, [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')], False @@ -1142,13 +1499,11 @@ class LargeFileSystem(object): if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'): return False contentTempFile = self.generateTempFile(contents) - compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False) - zf = zipfile.ZipFile(compressedContentFile.name, mode='w') - zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED) - zf.close() - compressedContentsSize = zf.infolist()[0].compress_size + compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=True) + with zipfile.ZipFile(compressedContentFile, mode='w') as zf: + zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED) + compressedContentsSize = zf.infolist()[0].compress_size os.remove(contentTempFile) - os.remove(compressedContentFile.name) if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'): return True return False @@ -1165,10 +1520,11 @@ class LargeFileSystem(object): def processContent(self, git_mode, relPath, contents): """Processes the content of git fast import. This method decides if a file is stored in the large file system and handles all necessary - steps.""" + steps. + """ if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath): contentTempFile = self.generateTempFile(contents) - (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile) + pointer_git_mode, contents, localLargeFile = self.generatePointer(contentTempFile) if pointer_git_mode: git_mode = pointer_git_mode if localLargeFile: @@ -1184,12 +1540,14 @@ class LargeFileSystem(object): sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile)) return (git_mode, contents) + class MockLFS(LargeFileSystem): """Mock large file system for testing.""" def generatePointer(self, contentFile): """The pointer content is the original content prefixed with "pointer-". - The local filename of the large file storage is derived from the file content. + The local filename of the large file storage is derived from the + file content. """ with open(contentFile, 'r') as f: content = next(f) @@ -1199,17 +1557,19 @@ class MockLFS(LargeFileSystem): return (gitMode, pointerContents, localLargeFile) def pushFile(self, localLargeFile): - """The remote filename of the large file storage is the same as the local - one but in a different directory. + """The remote filename of the large file storage is the same as the + local one but in a different directory. """ remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote') if not os.path.exists(remotePath): os.makedirs(remotePath) shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile))) + class GitLFS(LargeFileSystem): """Git LFS as backend for the git-p4 large file system. - See https://git-lfs.github.com/ for details.""" + See https://git-lfs.github.com/ for details. + """ def __init__(self, *args): LargeFileSystem.__init__(self, *args) @@ -1228,7 +1588,7 @@ class GitLFS(LargeFileSystem): ['git', 'lfs', 'pointer', '--file=' + contentFile], stdout=subprocess.PIPE ) - pointerFile = pointerProcess.stdout.read() + pointerFile = decode_text_stream(pointerProcess.stdout.read()) if pointerProcess.wait(): os.remove(contentFile) die('git-lfs pointer command failed. Did you install the extension?') @@ -1241,9 +1601,15 @@ class GitLFS(LargeFileSystem): pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile) oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1) + # if someone use external lfs.storage ( not in local repo git ) + lfs_path = gitConfig('lfs.storage') + if not lfs_path: + lfs_path = 'lfs' + if not os.path.isabs(lfs_path): + lfs_path = os.path.join(os.getcwd(), '.git', lfs_path) localLargeFile = os.path.join( - os.getcwd(), - '.git', 'lfs', 'objects', oid[:2], oid[2:4], + lfs_path, + 'objects', oid[:2], oid[2:4], oid, ) # LFS Spec states that pointer files should not have the executable bit set. @@ -1289,18 +1655,23 @@ class GitLFS(LargeFileSystem): else: return LargeFileSystem.processContent(self, git_mode, relPath, contents) + class Command: + delete_actions = ("delete", "move/delete", "purge") + add_actions = ("add", "branch", "move/add") + def __init__(self): self.usage = "usage: %prog [options]" self.needsGit = True self.verbose = False - # This is required for the "append" cloneExclude action + # This is required for the "append" update_shelve action def ensure_value(self, attr, value): if not hasattr(self, attr) or getattr(self, attr) is None: setattr(self, attr, value) return getattr(self, attr) + class P4UserMap: def __init__(self): self.userMapFromPerforceServer = False @@ -1310,15 +1681,15 @@ class P4UserMap: if self.myP4UserId: return self.myP4UserId - results = p4CmdList("user -o") + results = p4CmdList(["user", "-o"]) for r in results: - if r.has_key('User'): + if 'User' in r: self.myP4UserId = r['User'] return r['User'] die("Could not find your p4 user id") def p4UserIsMe(self, p4User): - # return True if the given p4 user is actually me + """Return True if the given p4 user is actually me.""" me = self.p4UserId() if not p4User or p4User != me: return False @@ -1335,10 +1706,16 @@ class P4UserMap: self.users = {} self.emails = {} - for output in p4CmdList("users"): - if not output.has_key("User"): + for output in p4CmdList(["users"]): + if "User" not in output: continue - self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">" + # "FullName" is bytes. "Email" on the other hand might be bytes + # or unicode string depending on whether we are running under + # python2 or python3. To support + # git-p4.metadataDecodingStrategy=fallback, self.users dict values + # are always bytes, ready to be written to git. + emailbytes = metadata_stream_to_writable_bytes(output["Email"]) + self.users[output["User"]] = output["FullName"] + b" <" + emailbytes + b">" self.emails[output["Email"]] = output["User"] mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE) @@ -1348,102 +1725,31 @@ class P4UserMap: user = mapUser[0][0] fullname = mapUser[0][1] email = mapUser[0][2] - self.users[user] = fullname + " <" + email + ">" + fulluser = fullname + " <" + email + ">" + self.users[user] = metadata_stream_to_writable_bytes(fulluser) self.emails[email] = user - s = '' + s = b'' for (key, val) in self.users.items(): - s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1)) + keybytes = metadata_stream_to_writable_bytes(key) + s += b"%s\t%s\n" % (keybytes.expandtabs(1), val.expandtabs(1)) - open(self.getUserCacheFilename(), "wb").write(s) + open(self.getUserCacheFilename(), 'wb').write(s) self.userMapFromPerforceServer = True def loadUserMapFromCache(self): self.users = {} self.userMapFromPerforceServer = False try: - cache = open(self.getUserCacheFilename(), "rb") + cache = open(self.getUserCacheFilename(), 'rb') lines = cache.readlines() cache.close() for line in lines: - entry = line.strip().split("\t") - self.users[entry[0]] = entry[1] + entry = line.strip().split(b"\t") + self.users[entry[0].decode('utf_8')] = entry[1] except IOError: self.getUserMapFromPerforceServer() -class P4Debug(Command): - def __init__(self): - Command.__init__(self) - self.options = [] - self.description = "A tool to debug the output of p4 -G." - self.needsGit = False - - def run(self, args): - j = 0 - for output in p4CmdList(args): - print 'Element: %d' % j - j += 1 - print output - return True - -class P4RollBack(Command): - def __init__(self): - Command.__init__(self) - self.options = [ - optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true") - ] - self.description = "A tool to debug the multi-branch import. Don't use :)" - self.rollbackLocalBranches = False - - def run(self, args): - if len(args) != 1: - return False - maxChange = int(args[0]) - - if "p4ExitCode" in p4Cmd("changes -m 1"): - die("Problems executing p4"); - - if self.rollbackLocalBranches: - refPrefix = "refs/heads/" - lines = read_pipe_lines("git rev-parse --symbolic --branches") - else: - refPrefix = "refs/remotes/" - lines = read_pipe_lines("git rev-parse --symbolic --remotes") - - for line in lines: - if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"): - line = line.strip() - ref = refPrefix + line - log = extractLogMessageFromGitCommit(ref) - settings = extractSettingsGitLog(log) - - depotPaths = settings['depot-paths'] - change = settings['change'] - - changed = False - - if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange) - for p in depotPaths]))) == 0: - print "Branch %s did not exist at change %s, deleting." % (ref, maxChange) - system("git update-ref -d %s `git rev-parse %s`" % (ref, ref)) - continue - - while change and int(change) > maxChange: - changed = True - if self.verbose: - print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange) - system("git update-ref %s \"%s^\"" % (ref, ref)) - log = extractLogMessageFromGitCommit(ref) - settings = extractSettingsGitLog(log) - - - depotPaths = settings['depot-paths'] - change = settings['change'] - - if changed: - print "%s rewound to %s" % (ref, change) - - return True class P4Submit(Command, P4UserMap): @@ -1477,8 +1783,40 @@ class P4Submit(Command, P4UserMap): "work from a local git branch that is not master"), optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true", help="Skip Perforce sync of p4/master after submit or shelve"), + optparse.make_option("--no-verify", dest="no_verify", action="store_true", + help="Bypass p4-pre-submit and p4-changelist hooks"), ] - self.description = "Submit changes from git to the perforce depot." + self.description = """Submit changes from git to the perforce depot.\n + The `p4-pre-submit` hook is executed if it exists and is executable. It + can be bypassed with the `--no-verify` command line option. The hook takes + no parameters and nothing from standard input. Exiting with a non-zero status + from this script prevents `git-p4 submit` from launching. + + One usage scenario is to run unit tests in the hook. + + The `p4-prepare-changelist` hook is executed right after preparing the default + changelist message and before the editor is started. It takes one parameter, + the name of the file that contains the changelist text. Exiting with a non-zero + status from the script will abort the process. + + The purpose of the hook is to edit the message file in place, and it is not + supressed by the `--no-verify` option. This hook is called even if + `--prepare-p4-only` is set. + + The `p4-changelist` hook is executed after the changelist message has been + edited by the user. It can be bypassed with the `--no-verify` option. It + takes a single parameter, the name of the file that holds the proposed + changelist text. Exiting with a non-zero status causes the command to abort. + + The hook is allowed to edit the changelist file and can be used to normalize + the text into some project standard format. It can also be used to refuse the + Submit after inspect the message file. + + The `p4-post-changelist` hook is invoked after the submit has successfully + occurred in P4. It takes no parameters and is meant primarily for notification + and cannot affect the outcome of the git p4 submit action. + """ + self.usage += " [name of git branch to submit into perforce depot]" self.origin = "" self.detectRenames = False @@ -1495,29 +1833,30 @@ class P4Submit(Command, P4UserMap): self.exportLabels = False self.p4HasMoveCommand = p4_has_move_command() self.branch = None + self.no_verify = False if gitConfig('git-p4.largeFileSystem'): die("Large file system not supported for git-p4 submit command. Please remove it from config.") def check(self): - if len(p4CmdList("opened ...")) > 0: + if len(p4CmdList(["opened", "..."])) > 0: die("You have files opened with perforce! Close them before starting the sync.") def separate_jobs_from_description(self, message): - """Extract and return a possible Jobs field in the commit - message. It goes into a separate section in the p4 change - specification. + """Extract and return a possible Jobs field in the commit message. It + goes into a separate section in the p4 change specification. - A jobs line starts with "Jobs:" and looks like a new field - in a form. Values are white-space separated on the same - line or on following lines that start with a tab. + A jobs line starts with "Jobs:" and looks like a new field in a + form. Values are white-space separated on the same line or on + following lines that start with a tab. - This does not parse and extract the full git commit message - like a p4 form. It just sees the Jobs: line as a marker - to pass everything from then on directly into the p4 form, - but outside the description section. + This does not parse and extract the full git commit message like a + p4 form. It just sees the Jobs: line as a marker to pass everything + from then on directly into the p4 form, but outside the description + section. - Return a tuple (stripped log message, jobs string).""" + Return a tuple (stripped log message, jobs string). + """ m = re.search(r'^Jobs:', message, re.MULTILINE) if m is None: @@ -1528,9 +1867,10 @@ class P4Submit(Command, P4UserMap): return (stripped_message, jobtext) def prepareLogMessage(self, template, message, jobs): - """Edits the template returned from "p4 change -o" to insert - the message in the Description field, and the jobs text in - the Jobs field.""" + """Edits the template returned from "p4 change -o" to insert the + message in the Description field, and the jobs text in the Jobs + field. + """ result = "" inDescriptionSection = False @@ -1559,98 +1899,103 @@ class P4Submit(Command, P4UserMap): return result - def patchRCSKeywords(self, file, pattern): - # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern - (handle, outFileName) = tempfile.mkstemp(dir='.') + def patchRCSKeywords(self, file, regexp): + """Attempt to zap the RCS keywords in a p4 controlled file matching the + given regex. + """ + handle, outFileName = tempfile.mkstemp(dir='.') try: - outFile = os.fdopen(handle, "w+") - inFile = open(file, "r") - regexp = re.compile(pattern, re.VERBOSE) - for line in inFile.readlines(): - line = regexp.sub(r'$\1$', line) - outFile.write(line) - inFile.close() - outFile.close() + with os.fdopen(handle, "wb") as outFile, open(file, "rb") as inFile: + for line in inFile.readlines(): + outFile.write(regexp.sub(br'$\1$', line)) # Forcibly overwrite the original file os.unlink(file) shutil.move(outFileName, file) except: # cleanup our temporary file os.unlink(outFileName) - print "Failed to strip RCS keywords in %s" % file + print("Failed to strip RCS keywords in %s" % file) raise - print "Patched up RCS keywords in %s" % file + print("Patched up RCS keywords in %s" % file) - def p4UserForCommit(self,id): - # Return the tuple (perforce user,git email) for a given git commit id + def p4UserForCommit(self, id): + """Return the tuple (perforce user,git email) for a given git commit + id. + """ self.getUserMapFromPerforceServer() gitEmail = read_pipe(["git", "log", "--max-count=1", "--format=%ae", id]) gitEmail = gitEmail.strip() - if not self.emails.has_key(gitEmail): - return (None,gitEmail) + if gitEmail not in self.emails: + return (None, gitEmail) else: - return (self.emails[gitEmail],gitEmail) + return (self.emails[gitEmail], gitEmail) - def checkValidP4Users(self,commits): - # check if any git authors cannot be mapped to p4 users + def checkValidP4Users(self, commits): + """Check if any git authors cannot be mapped to p4 users.""" for id in commits: - (user,email) = self.p4UserForCommit(id) + user, email = self.p4UserForCommit(id) if not user: msg = "Cannot find p4 user for email %s in commit %s." % (email, id) if gitConfigBool("git-p4.allowMissingP4Users"): - print "%s" % msg + print("%s" % msg) else: die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg) def lastP4Changelist(self): - # Get back the last changelist number submitted in this client spec. This - # then gets used to patch up the username in the change. If the same - # client spec is being used by multiple processes then this might go - # wrong. - results = p4CmdList("client -o") # find the current client + """Get back the last changelist number submitted in this client spec. + + This then gets used to patch up the username in the change. If the + same client spec is being used by multiple processes then this might + go wrong. + """ + results = p4CmdList(["client", "-o"]) # find the current client client = None for r in results: - if r.has_key('Client'): + if 'Client' in r: client = r['Client'] break if not client: die("could not get client spec") results = p4CmdList(["changes", "-c", client, "-m", "1"]) for r in results: - if r.has_key('change'): + if 'change' in r: return r['change'] die("Could not get changelist number for last submit - cannot patch up user details") def modifyChangelistUser(self, changelist, newUser): - # fixup the user field of a changelist after it has been submitted. - changes = p4CmdList("change -o %s" % changelist) + """Fixup the user field of a changelist after it has been submitted.""" + changes = p4CmdList(["change", "-o", changelist]) if len(changes) != 1: die("Bad output from p4 change modifying %s to user %s" % (changelist, newUser)) c = changes[0] - if c['User'] == newUser: return # nothing to do + if c['User'] == newUser: + # Nothing to do + return c['User'] = newUser - input = marshal.dumps(c) + # p4 does not understand format version 3 and above + input = marshal.dumps(c, 2) - result = p4CmdList("change -f -i", stdin=input) + result = p4CmdList(["change", "-f", "-i"], stdin=input) for r in result: - if r.has_key('code'): + if 'code' in r: if r['code'] == 'error': die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data'])) - if r.has_key('data'): + if 'data' in r: print("Updated user field for changelist %s to %s" % (changelist, newUser)) return die("Could not modify user field of changelist %s to %s" % (changelist, newUser)) def canChangeChangelists(self): - # check to see if we have p4 admin or super-user permissions, either of - # which are required to modify changelists. + """Check to see if we have p4 admin or super-user permissions, either + of which are required to modify changelists. + """ results = p4CmdList(["protects", self.depotPath]) for r in results: - if r.has_key('perm'): + if 'perm' in r: if r['perm'] == 'admin': return 1 if r['perm'] == 'super': @@ -1659,13 +2004,15 @@ class P4Submit(Command, P4UserMap): def prepareSubmitTemplate(self, changelist=None): """Run "p4 change -o" to grab a change specification template. + This does not use "p4 -G", as it is nice to keep the submission template in original order, since a human might edit it. Remove lines in the Files section that show changes to files - outside the depot path we're committing into.""" + outside the depot path we're committing into. + """ - [upstream, settings] = findUpstreamBranchPoint() + upstream, settings = findUpstreamBranchPoint() template = """\ # A Perforce Change Specification. @@ -1690,16 +2037,16 @@ class P4Submit(Command, P4UserMap): if changelist: args.append(str(changelist)) for entry in p4CmdList(args): - if not entry.has_key('code'): + if 'code' not in entry: continue if entry['code'] == 'stat': change_entry = entry break if not change_entry: die('Failed to decode output of p4 change -o') - for key, value in change_entry.iteritems(): + for key, value in change_entry.items(): if key.startswith('File'): - if settings.has_key('depot-paths'): + if 'depot-paths' in settings: if not [p for p in settings['depot-paths'] if p4PathStartsWith(value, p)]: continue @@ -1710,7 +2057,7 @@ class P4Submit(Command, P4UserMap): continue # Output in the order expected by prepareLogMessage for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']: - if not change_entry.has_key(key): + if key not in change_entry: continue template += '\n' template += key + ':' @@ -1726,8 +2073,10 @@ class P4Submit(Command, P4UserMap): return template def edit_template(self, template_file): - """Invoke the editor to let the user change the submission - message. Return true if okay to continue with the submit.""" + """Invoke the editor to let the user change the submission message. + + Return true if okay to continue with the submit. + """ # if configured to skip the editing part, just submit if gitConfigBool("git-p4.skipSubmitEdit"): @@ -1738,10 +2087,10 @@ class P4Submit(Command, P4UserMap): mtime = os.stat(template_file).st_mtime # invoke the editor - if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""): + if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""): editor = os.environ.get("P4EDITOR") else: - editor = read_pipe("git var GIT_EDITOR").strip() + editor = read_pipe(["git", "var", "GIT_EDITOR"]).strip() system(["sh", "-c", ('%s "$@"' % editor), editor, template_file]) # If the file was not saved, prompt to see if this patch should @@ -1753,16 +2102,15 @@ class P4Submit(Command, P4UserMap): if os.stat(template_file).st_mtime > mtime: return True - while True: - response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ") - if response == 'y': - return True - if response == 'n': - return False + response = prompt("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ") + if response == 'y': + return True + if response == 'n': + return False def get_diff_description(self, editedFiles, filesToAdd, symlinks): # diff - if os.environ.has_key("P4DIFF"): + if "P4DIFF" in os.environ: del(os.environ["P4DIFF"]) diff = "" for editedFile in editedFiles: @@ -1783,8 +2131,13 @@ class P4Submit(Command, P4UserMap): newdiff += "+%s\n" % os.readlink(newFile) else: f = open(newFile, "r") - for line in f.readlines(): - newdiff += "+" + line + try: + for line in f.readlines(): + newdiff += "+" + line + except UnicodeDecodeError: + # Found non-text data and skip, since diff description + # should only include text + pass f.close() return (diff + newdiff).replace('\r\n', '\n') @@ -1792,12 +2145,13 @@ class P4Submit(Command, P4UserMap): def applyCommit(self, id): """Apply one commit, return True if it succeeded.""" - print "Applying", read_pipe(["git", "show", "-s", - "--format=format:%h %s", id]) + print("Applying", read_pipe(["git", "show", "-s", + "--format=format:%h %s", id])) - (p4User, gitEmail) = self.p4UserForCommit(id) + p4User, gitEmail = self.p4UserForCommit(id) - diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id)) + diff = read_pipe_lines( + ["git", "diff-tree", "-r"] + self.diffOpts + ["{}^".format(id), id]) filesToAdd = set() filesToChangeType = set() filesToDelete = set() @@ -1825,7 +2179,7 @@ class P4Submit(Command, P4UserMap): filesToDelete.remove(path) dst_mode = int(diff['dst_mode'], 8) - if dst_mode == 0120000: + if dst_mode == 0o120000: symlinks.add(path) elif modifier == "D": @@ -1834,6 +2188,7 @@ class P4Submit(Command, P4UserMap): filesToAdd.remove(path) elif modifier == "C": src, dest = diff['src'], diff['dst'] + all_files.append(dest) p4_integrate(src, dest) pureRenameCopy.add(dest) if diff['src_sha1'] != diff['dst_sha1']: @@ -1850,6 +2205,7 @@ class P4Submit(Command, P4UserMap): editedFiles.add(dest) elif modifier == "R": src, dest = diff['src'], diff['dst'] + all_files.append(dest) if self.p4HasMoveCommand: p4_edit(src) # src must be open before move p4_move(src, dest) # opens for (move/delete, move/add) @@ -1880,34 +2236,36 @@ class P4Submit(Command, P4UserMap): applyPatchCmd = patchcmd + "--check --apply -" patch_succeeded = True + if verbose: + print("TryPatch: %s" % tryPatchCmd) + if os.system(tryPatchCmd) != 0: fixed_rcs_keywords = False patch_succeeded = False - print "Unfortunately applying the change failed!" + print("Unfortunately applying the change failed!") # Patch failed, maybe it's just RCS keyword woes. Look through # the patch to see if that's possible. if gitConfigBool("git-p4.attemptRCSCleanup"): file = None - pattern = None kwfiles = {} for file in editedFiles | filesToDelete: # did this file's delta contain RCS keywords? - pattern = p4_keywords_regexp_for_file(file) - - if pattern: + regexp = p4_keywords_regexp_for_file(file) + if regexp: # this file is a possibility...look for RCS keywords. - regexp = re.compile(pattern, re.VERBOSE) - for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]): + for line in read_pipe_lines( + ["git", "diff", "%s^..%s" % (id, id), file], + raw=True): if regexp.search(line): if verbose: - print "got keyword match on %s in %s in %s" % (pattern, line, file) - kwfiles[file] = pattern + print("got keyword match on %s in %s in %s" % (regexp.pattern, line, file)) + kwfiles[file] = regexp break - for file in kwfiles: + for file, regexp in kwfiles.items(): if verbose: - print "zapping %s with %s" % (line,pattern) + print("zapping %s with %s" % (line, regexp.pattern)) # File is being deleted, so not open in p4. Must # disable the read-only bit on windows. if self.isWindows and file not in editedFiles: @@ -1916,9 +2274,10 @@ class P4Submit(Command, P4UserMap): fixed_rcs_keywords = True if fixed_rcs_keywords: - print "Retrying the patch with RCS keywords cleaned up" + print("Retrying the patch with RCS keywords cleaned up") if os.system(tryPatchCmd) == 0: patch_succeeded = True + print("Patch succeesed this time with RCS keywords cleaned") if not patch_succeeded: for f in editedFiles: @@ -1928,7 +2287,7 @@ class P4Submit(Command, P4UserMap): # # Apply the patch for real, and do add/delete/+x handling. # - system(applyPatchCmd) + system(applyPatchCmd, shell=True) for f in filesToChangeType: p4_edit(f, "-t", "auto") @@ -1954,13 +2313,13 @@ class P4Submit(Command, P4UserMap): # logMessage = extractLogMessageFromGitCommit(id) logMessage = logMessage.strip() - (logMessage, jobs) = self.separate_jobs_from_description(logMessage) + logMessage, jobs = self.separate_jobs_from_description(logMessage) template = self.prepareSubmitTemplate(update_shelve) submitTemplate = self.prepareLogMessage(template, logMessage, jobs) if self.preserveUser: - submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User + submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User if self.checkAuthorship and not self.p4UserIsMe(p4User): submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail @@ -1972,62 +2331,80 @@ class P4Submit(Command, P4UserMap): submitTemplate += separatorLine submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks) - (handle, fileName) = tempfile.mkstemp() + handle, fileName = tempfile.mkstemp() tmpFile = os.fdopen(handle, "w+b") if self.isWindows: submitTemplate = submitTemplate.replace("\n", "\r\n") - tmpFile.write(submitTemplate) + tmpFile.write(encode_text_stream(submitTemplate)) tmpFile.close() - if self.prepare_p4_only: - # - # Leave the p4 tree prepared, and the submit template around - # and let the user decide what to do next - # - print - print "P4 workspace prepared for submission." - print "To submit or revert, go to client workspace" - print " " + self.clientPath - print - print "To submit, use \"p4 submit\" to write a new description," - print "or \"p4 submit -i <%s\" to use the one prepared by" \ - " \"git p4\"." % fileName - print "You can delete the file \"%s\" when finished." % fileName - - if self.preserveUser and p4User and not self.p4UserIsMe(p4User): - print "To preserve change ownership by user %s, you must\n" \ - "do \"p4 change -f <change>\" after submitting and\n" \ - "edit the User field." - if pureRenameCopy: - print "After submitting, renamed files must be re-synced." - print "Invoke \"p4 sync -f\" on each of these files:" - for f in pureRenameCopy: - print " " + f - - print - print "To revert the changes, use \"p4 revert ...\", and delete" - print "the submit template file \"%s\"" % fileName - if filesToAdd: - print "Since the commit adds new files, they must be deleted:" - for f in filesToAdd: - print " " + f - print - return True - - # - # Let the user edit the change description, then submit it. - # submitted = False try: + # Allow the hook to edit the changelist text before presenting it + # to the user. + if not run_git_hook("p4-prepare-changelist", [fileName]): + return False + + if self.prepare_p4_only: + # + # Leave the p4 tree prepared, and the submit template around + # and let the user decide what to do next + # + submitted = True + print("") + print("P4 workspace prepared for submission.") + print("To submit or revert, go to client workspace") + print(" " + self.clientPath) + print("") + print("To submit, use \"p4 submit\" to write a new description,") + print("or \"p4 submit -i <%s\" to use the one prepared by" + " \"git p4\"." % fileName) + print("You can delete the file \"%s\" when finished." % fileName) + + if self.preserveUser and p4User and not self.p4UserIsMe(p4User): + print("To preserve change ownership by user %s, you must\n" + "do \"p4 change -f <change>\" after submitting and\n" + "edit the User field.") + if pureRenameCopy: + print("After submitting, renamed files must be re-synced.") + print("Invoke \"p4 sync -f\" on each of these files:") + for f in pureRenameCopy: + print(" " + f) + + print("") + print("To revert the changes, use \"p4 revert ...\", and delete") + print("the submit template file \"%s\"" % fileName) + if filesToAdd: + print("Since the commit adds new files, they must be deleted:") + for f in filesToAdd: + print(" " + f) + print("") + sys.stdout.flush() + return True + if self.edit_template(fileName): + if not self.no_verify: + if not run_git_hook("p4-changelist", [fileName]): + print("The p4-changelist hook failed.") + sys.stdout.flush() + return False + # read the edited message and submit tmpFile = open(fileName, "rb") - message = tmpFile.read() + message = decode_text_stream(tmpFile.read()) tmpFile.close() if self.isWindows: message = message.replace("\r\n", "\n") - submitTemplate = message[:message.index(separatorLine)] + if message.find(separatorLine) != -1: + submitTemplate = message[:message.index(separatorLine)] + else: + submitTemplate = message + + if len(submitTemplate.strip()) == 0: + print("Changelist is empty, aborting this changelist.") + sys.stdout.flush() + return False if update_shelve: p4_write_pipe(['shelve', '-r', '-i'], submitTemplate) @@ -2050,25 +2427,30 @@ class P4Submit(Command, P4UserMap): submitted = True + run_git_hook("p4-post-changelist") finally: - # skip this patch + # Revert changes if we skip this patch if not submitted or self.shelve: if self.shelve: - print ("Reverting shelved files.") + print("Reverting shelved files.") else: - print ("Submission cancelled, undoing p4 changes.") + print("Submission cancelled, undoing p4 changes.") + sys.stdout.flush() for f in editedFiles | filesToDelete: p4_revert(f) for f in filesToAdd: p4_revert(f) os.remove(f) - os.remove(fileName) + if not self.prepare_p4_only: + os.remove(fileName) return submitted - # Export git tags as p4 labels. Create a p4 label and then tag - # with that. def exportGitTags(self, gitTags): + """Export git tags as p4 labels. Create a p4 label and then tag with + that. + """ + validLabelRegexp = gitConfig("git-p4.labelExportRegexp") if len(validLabelRegexp) == 0: validLabelRegexp = defaultLabelRegexp @@ -2078,17 +2460,17 @@ class P4Submit(Command, P4UserMap): if not m.match(name): if verbose: - print "tag %s does not match regexp %s" % (name, validLabelRegexp) + print("tag %s does not match regexp %s" % (name, validLabelRegexp)) continue # Get the p4 commit this corresponds to logMessage = extractLogMessageFromGitCommit(name) values = extractSettingsGitLog(logMessage) - if not values.has_key('change'): + if 'change' not in values: # a tag pointing to something not sent to p4; ignore if verbose: - print "git tag %s does not give a p4 commit" % name + print("git tag %s does not give a p4 commit" % name) continue else: changelist = values['change'] @@ -2114,7 +2496,7 @@ class P4Submit(Command, P4UserMap): # Create the label - use the same view as the client spec we are using clientSpec = getClientSpec() - labelTemplate = "Label: %s\n" % name + labelTemplate = "Label: %s\n" % name labelTemplate += "Description:\n" for b in body: labelTemplate += "\t" + b + "\n" @@ -2123,10 +2505,10 @@ class P4Submit(Command, P4UserMap): labelTemplate += "\t%s\n" % depot_side if self.dry_run: - print "Would create p4 label %s for tag" % name + print("Would create p4 label %s for tag" % name) elif self.prepare_p4_only: - print "Not creating p4 label %s for tag due to option" \ - " --prepare-p4-only" % name + print("Not creating p4 label %s for tag due to option" + " --prepare-p4-only" % name) else: p4_write_pipe(["label", "-i"], labelTemplate) @@ -2135,7 +2517,7 @@ class P4Submit(Command, P4UserMap): ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings]) if verbose: - print "created p4 label for tag %s" % name + print("created p4 label for tag %s" % name) def run(self, args): if len(args) == 0: @@ -2156,7 +2538,7 @@ class P4Submit(Command, P4UserMap): if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","): die("%s is not in git-p4.allowSubmit" % self.master) - [upstream, settings] = findUpstreamBranchPoint() + upstream, settings = findUpstreamBranchPoint() self.depotPath = settings['depot-paths'][0] if len(self.origin) == 0: self.origin = upstream @@ -2179,10 +2561,10 @@ class P4Submit(Command, P4UserMap): self.conflict_behavior = val if self.verbose: - print "Origin branch is " + self.origin + print("Origin branch is " + self.origin) if len(self.depotPath) == 0: - print "Internal error: cannot locate perforce depot path from existing branches" + print("Internal error: cannot locate perforce depot path from existing branches") sys.exit(128) self.useClientSpec = False @@ -2203,7 +2585,7 @@ class P4Submit(Command, P4UserMap): if self.clientPath == "": die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath) - print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath) + print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)) self.oldWorkingDirectory = os.getcwd() # ensure the clientPath exists @@ -2214,9 +2596,9 @@ class P4Submit(Command, P4UserMap): chdir(self.clientPath, is_client_path=True) if self.dry_run: - print "Would synchronize p4 checkout in %s" % self.clientPath + print("Would synchronize p4 checkout in %s" % self.clientPath) else: - print "Synchronizing p4 checkout..." + print("Synchronizing p4 checkout...") if new_client_dir: # old one was destroyed, and maybe nobody told p4 p4_sync("...", "-f") @@ -2257,17 +2639,17 @@ class P4Submit(Command, P4UserMap): # if self.detectRenames: # command-line -M arg - self.diffOpts = "-M" + self.diffOpts = ["-M"] else: # If not explicitly set check the config variable detectRenames = gitConfig("git-p4.detectRenames") if detectRenames.lower() == "false" or detectRenames == "": - self.diffOpts = "" + self.diffOpts = [] elif detectRenames.lower() == "true": - self.diffOpts = "-M" + self.diffOpts = ["-M"] else: - self.diffOpts = "-M%s" % detectRenames + self.diffOpts = ["-M{}".format(detectRenames)] # no command-line arg for -C or --find-copies-harder, just # config variables @@ -2275,66 +2657,70 @@ class P4Submit(Command, P4UserMap): if detectCopies.lower() == "false" or detectCopies == "": pass elif detectCopies.lower() == "true": - self.diffOpts += " -C" + self.diffOpts.append("-C") else: - self.diffOpts += " -C%s" % detectCopies + self.diffOpts.append("-C{}".format(detectCopies)) if gitConfigBool("git-p4.detectCopiesHarder"): - self.diffOpts += " --find-copies-harder" + self.diffOpts.append("--find-copies-harder") num_shelves = len(self.update_shelve) if num_shelves > 0 and num_shelves != len(commits): sys.exit("number of commits (%d) must match number of shelved changelist (%d)" % (len(commits), num_shelves)) + if not self.no_verify: + try: + if not run_git_hook("p4-pre-submit"): + print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nYou can skip " + "this pre-submission check by adding\nthe command line option '--no-verify', " + "however,\nthis will also skip the p4-changelist hook as well.") + sys.exit(1) + except Exception as e: + print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nThe hook failed " + "with the error '{0}'".format(e.message)) + sys.exit(1) + # # Apply the commits, one at a time. On failure, ask if should # continue to try the rest of the patches, or quit. # if self.dry_run: - print "Would apply" + print("Would apply") applied = [] last = len(commits) - 1 for i, commit in enumerate(commits): if self.dry_run: - print " ", read_pipe(["git", "show", "-s", - "--format=format:%h %s", commit]) + print(" ", read_pipe(["git", "show", "-s", + "--format=format:%h %s", commit])) ok = True else: ok = self.applyCommit(commit) if ok: applied.append(commit) - else: - if self.prepare_p4_only and i < last: - print "Processing only the first commit due to option" \ - " --prepare-p4-only" + if self.prepare_p4_only: + if i < last: + print("Processing only the first commit due to option" + " --prepare-p4-only") break + else: if i < last: - quit = False - while True: - # prompt for what to do, or use the option/variable - if self.conflict_behavior == "ask": - print "What do you want to do?" - response = raw_input("[s]kip this commit but apply" - " the rest, or [q]uit? ") - if not response: - continue - elif self.conflict_behavior == "skip": - response = "s" - elif self.conflict_behavior == "quit": - response = "q" - else: - die("Unknown conflict_behavior '%s'" % - self.conflict_behavior) - - if response[0] == "s": - print "Skipping this commit, but applying the rest" - break - if response[0] == "q": - print "Quitting" - quit = True - break - if quit: + # prompt for what to do, or use the option/variable + if self.conflict_behavior == "ask": + print("What do you want to do?") + response = prompt("[s]kip this commit but apply the rest, or [q]uit? ") + elif self.conflict_behavior == "skip": + response = "s" + elif self.conflict_behavior == "quit": + response = "q" + else: + die("Unknown conflict_behavior '%s'" % + self.conflict_behavior) + + if response == "s": + print("Skipping this commit, but applying the rest") + if response == "q": + print("Quitting") break chdir(self.oldWorkingDirectory) @@ -2344,7 +2730,7 @@ class P4Submit(Command, P4UserMap): elif self.prepare_p4_only: pass elif len(commits) == len(applied): - print ("All commits {0}!".format(shelved_applied)) + print("All commits {0}!".format(shelved_applied)) sync = P4Sync() if self.branch: @@ -2360,17 +2746,17 @@ class P4Submit(Command, P4UserMap): else: if len(applied) == 0: - print ("No commits {0}.".format(shelved_applied)) + print("No commits {0}.".format(shelved_applied)) else: - print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize())) + print("{0} only the commits marked with '*':".format(shelved_applied.capitalize())) for c in commits: if c in applied: star = "*" else: star = " " - print star, read_pipe(["git", "show", "-s", - "--format=format:%h %s", c]) - print "You will have to do 'git p4 sync' and rebase." + print(star, read_pipe(["git", "show", "-s", + "--format=format:%h %s", c])) + print("You will have to do 'git p4 sync' and rebase.") if gitConfigBool("git-p4.exportLabels"): self.exportLabels = True @@ -2384,13 +2770,15 @@ class P4Submit(Command, P4UserMap): # exit with error unless everything applied perfectly if len(commits) != len(applied): - sys.exit(1) + sys.exit(1) return True + class View(object): - """Represent a p4 view ("p4 help views"), and map files in a - repo according to the view.""" + """Represent a p4 view ("p4 help views"), and map files in a repo according + to the view. + """ def __init__(self, client_name): self.mappings = [] @@ -2399,9 +2787,10 @@ class View(object): self.client_spec_path_cache = {} def append(self, view_line): - """Parse a view line, splitting it into depot and client - sides. Append to self.mappings, preserving order. This - is only needed for tag creation.""" + """Parse a view line, splitting it into depot and client sides. Append + to self.mappings, preserving order. This is only needed for tag + creation. + """ # Split the view line into exactly two words. P4 enforces # structure on these lines that simplifies this quite a bit. @@ -2444,16 +2833,16 @@ class View(object): def convert_client_path(self, clientFile): # chop off //client/ part to make it relative - if not clientFile.startswith(self.client_prefix): + if not decode_path(clientFile).startswith(self.client_prefix): die("No prefix '%s' on clientFile '%s'" % (self.client_prefix, clientFile)) return clientFile[len(self.client_prefix):] def update_client_spec_path_cache(self, files): - """ Caching file paths by "p4 where" batch query """ + """Caching file paths by "p4 where" batch query.""" # List depot file paths exclude that already cached - fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache] + fileArgs = [f['path'] for f in files if decode_path(f['path']) not in self.client_spec_path_cache] if len(fileArgs) == 0: return # All files in cache @@ -2468,21 +2857,25 @@ class View(object): if "unmap" in res: # it will list all of them, but only one not unmap-ped continue + depot_path = decode_path(res['depotFile']) if gitConfigBool("core.ignorecase"): - res['depotFile'] = res['depotFile'].lower() - self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"]) + depot_path = depot_path.lower() + self.client_spec_path_cache[depot_path] = self.convert_client_path(res["clientFile"]) # not found files or unmap files set to "" for depotFile in fileArgs: + depotFile = decode_path(depotFile) if gitConfigBool("core.ignorecase"): depotFile = depotFile.lower() if depotFile not in self.client_spec_path_cache: - self.client_spec_path_cache[depotFile] = "" + self.client_spec_path_cache[depotFile] = b'' def map_in_client(self, depot_path): - """Return the relative location in the client where this - depot file should live. Returns "" if the file should - not be mapped in the client.""" + """Return the relative location in the client where this depot file + should live. + + Returns "" if the file should not be mapped in the client. + """ if gitConfigBool("core.ignorecase"): depot_path = depot_path.lower() @@ -2490,11 +2883,17 @@ class View(object): if depot_path in self.client_spec_path_cache: return self.client_spec_path_cache[depot_path] - die( "Error: %s is not found in client spec path" % depot_path ) + die("Error: %s is not found in client spec path" % depot_path) return "" + +def cloneExcludeCallback(option, opt_str, value, parser): + # prepend "/" because the first "/" was consumed as part of the option itself. + # ("-//depot/A/..." becomes "/depot/A/..." after option parsing) + parser.values.cloneExclude += ["/" + re.sub(r"\.\.\.$", "", value)] + + class P4Sync(Command, P4UserMap): - delete_actions = ( "delete", "move/delete", "purge" ) def __init__(self): Command.__init__(self) @@ -2517,7 +2916,7 @@ class P4Sync(Command, P4UserMap): optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true', help="Only sync files that are included in the Perforce Client Spec"), optparse.make_option("-/", dest="cloneExclude", - action="append", type="string", + action="callback", callback=cloneExcludeCallback, type="string", help="exclude depot path"), ] self.description = """Imports from Perforce into a git repository.\n @@ -2574,41 +2973,34 @@ class P4Sync(Command, P4UserMap): self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60)) self.labels = {} - # Force a checkpoint in fast-import and wait for it to finish def checkpoint(self): + """Force a checkpoint in fast-import and wait for it to finish.""" self.gitStream.write("checkpoint\n\n") self.gitStream.write("progress checkpoint\n\n") + self.gitStream.flush() out = self.gitOutput.readline() if self.verbose: - print "checkpoint finished: " + out - - def cmp_shelved(self, path, filerev, revision): - """ Determine if a path at revision #filerev is the same as the file - at revision @revision for a shelved changelist. If they don't match, - unshelving won't be safe (we will get other changes mixed in). + print("checkpoint finished: " + out) - This is comparing the revision that the shelved changelist is *based* on, not - the shelved changelist itself. - """ - ret = p4Cmd(["diff2", "{0}#{1}".format(path, filerev), "{0}@{1}".format(path, revision)]) - if verbose: - print("p4 diff2 path %s filerev %s revision %s => %s" % (path, filerev, revision, ret)) - return ret["status"] == "identical" + def isPathWanted(self, path): + for p in self.cloneExclude: + if p.endswith("/"): + if p4PathStartsWith(path, p): + return False + # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2" + elif path.lower() == p.lower(): + return False + for p in self.depotPaths: + if p4PathStartsWith(path, decode_path(p)): + return True + return False - def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0, origin_revision = 0): - self.cloneExclude = [re.sub(r"\.\.\.$", "", path) - for path in self.cloneExclude] + def extractFilesFromCommit(self, commit, shelved=False, shelved_cl=0): files = [] fnum = 0 - while commit.has_key("depotFile%s" % fnum): - path = commit["depotFile%s" % fnum] - - if [p for p in self.cloneExclude - if p4PathStartsWith(path, p)]: - found = False - else: - found = [p for p in self.depotPaths - if p4PathStartsWith(path, p)] + while "depotFile%s" % fnum in commit: + path = commit["depotFile%s" % fnum] + found = self.isPathWanted(decode_path(path)) if not found: fnum = fnum + 1 continue @@ -2620,17 +3012,6 @@ class P4Sync(Command, P4UserMap): file["type"] = commit["type%s" % fnum] if shelved: file["shelved_cl"] = int(shelved_cl) - - # For shelved changelists, check that the revision of each file that the - # shelve was based on matches the revision that we are using for the - # starting point for git-fast-import (self.initialParent). Otherwise - # the resulting diff will contain deltas from multiple commits. - - if file["action"] != "add" and \ - not self.cmp_shelved(path, file["rev"], origin_revision): - sys.exit("change {0} not based on {1} for {2}, cannot unshelve".format( - commit["change"], self.initialParent, path)) - files.append(file) fnum = fnum + 1 return files @@ -2638,25 +3019,25 @@ class P4Sync(Command, P4UserMap): def extractJobsFromCommit(self, commit): jobs = [] jnum = 0 - while commit.has_key("job%s" % jnum): + while "job%s" % jnum in commit: job = commit["job%s" % jnum] jobs.append(job) jnum = jnum + 1 return jobs def stripRepoPath(self, path, prefixes): - """When streaming files, this is called to map a p4 depot path - to where it should go in git. The prefixes are either - self.depotPaths, or self.branchPrefixes in the case of - branch detection.""" + """When streaming files, this is called to map a p4 depot path to where + it should go in git. The prefixes are either self.depotPaths, or + self.branchPrefixes in the case of branch detection. + """ if self.useClientSpec: # branch detection moves files up a level (the branch name) # from what client spec interpretation gives - path = self.clientSpecDirs.map_in_client(path) + path = decode_path(self.clientSpecDirs.map_in_client(path)) if self.detectBranches: for b in self.knownBranches: - if path.startswith(b + "/"): + if p4PathStartsWith(path, b + "/"): path = path[len(b)+1:] elif self.keepRepoPath: @@ -2677,8 +3058,9 @@ class P4Sync(Command, P4UserMap): return path def splitFilesIntoBranches(self, commit): - """Look at each depotFile in the commit to figure out to what - branch it belongs.""" + """Look at each depotFile in the commit to figure out to what branch it + belongs. + """ if self.clientSpecDirs: files = self.extractFilesFromCommit(commit) @@ -2686,16 +3068,16 @@ class P4Sync(Command, P4UserMap): branches = {} fnum = 0 - while commit.has_key("depotFile%s" % fnum): - path = commit["depotFile%s" % fnum] - found = [p for p in self.depotPaths - if p4PathStartsWith(path, p)] + while "depotFile%s" % fnum in commit: + raw_path = commit["depotFile%s" % fnum] + path = decode_path(raw_path) + found = self.isPathWanted(path) if not found: fnum = fnum + 1 continue file = {} - file["path"] = path + file["path"] = raw_path file["rev"] = commit["rev%s" % fnum] file["action"] = commit["action%s" % fnum] file["type"] = commit["type%s" % fnum] @@ -2704,14 +3086,14 @@ class P4Sync(Command, P4UserMap): # start with the full relative path where this file would # go in a p4 client if self.useClientSpec: - relPath = self.clientSpecDirs.map_in_client(path) + relPath = decode_path(self.clientSpecDirs.map_in_client(path)) else: relPath = self.stripRepoPath(path, self.depotPaths) for branch in self.knownBranches.keys(): # add a trailing slash so that a commit into qt/4.2foo # doesn't end up in qt/4.2, e.g. - if relPath.startswith(branch + "/"): + if p4PathStartsWith(relPath, branch + "/"): if branch not in branches: branches[branch] = [] branches[branch].append(file) @@ -2720,7 +3102,7 @@ class P4Sync(Command, P4UserMap): return branches def writeToGitStream(self, gitMode, relPath, contents): - self.gitStream.write('M %s inline %s\n' % (gitMode, relPath)) + self.gitStream.write(encode_text_stream(u'M {} inline {}\n'.format(gitMode, relPath))) self.gitStream.write('data %d\n' % sum(len(d) for d in contents)) for d in contents: self.gitStream.write(d) @@ -2735,21 +3117,29 @@ class P4Sync(Command, P4UserMap): encoding = gitConfig('git-p4.pathEncoding') path = path.decode(encoding, 'replace').encode('utf8', 'replace') if self.verbose: - print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path) + print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)) return path - # output one file from the P4 stream - # - helper for streamP4Files - def streamOneP4File(self, file, contents): - relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes) - relPath = self.encodeWithUTF8(relPath) + """Output one file from the P4 stream. + + This is a helper for streamP4Files(). + """ + + file_path = file['depotFile'] + relPath = self.stripRepoPath(decode_path(file_path), self.branchPrefixes) + if verbose: - size = int(self.stream_file['fileSize']) - sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024)) + if 'fileSize' in self.stream_file: + size = int(self.stream_file['fileSize']) + else: + # Deleted files don't get a fileSize apparently + size = 0 + sys.stdout.write('\r%s --> %s (%s)\n' % ( + file_path, relPath, format_size_human_readable(size))) sys.stdout.flush() - (type_base, type_mods) = split_p4_type(file["type"]) + type_base, type_mods = split_p4_type(file["type"]) git_mode = "100644" if "x" in type_mods: @@ -2758,13 +3148,13 @@ class P4Sync(Command, P4UserMap): git_mode = "120000" # p4 print on a symlink sometimes contains "target\n"; # if it does, remove the newline - data = ''.join(contents) + data = ''.join(decode_text_stream(c) for c in contents) if not data: # Some version of p4 allowed creating a symlink that pointed # to nothing. This causes p4 errors when checking out such # a change, and errors here too. Work around it by ignoring # the bad symlink; hopefully a future change fixes it. - print "\nIgnoring empty symlink in %s" % file['depotFile'] + print("\nIgnoring empty symlink in %s" % file_path) return elif data[-1] == '\n': contents = [data[:-1]] @@ -2783,7 +3173,7 @@ class P4Sync(Command, P4UserMap): # just the native "NT" type. # try: - text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])]) + text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (decode_path(file['depotFile']), file['change'])], raw=True) except Exception as e: if 'Translation of file content failed' in str(e): type_base = 'binary' @@ -2791,8 +3181,8 @@ class P4Sync(Command, P4UserMap): raise e else: if p4_version_string().find('/NT') >= 0: - text = text.replace('\r\n', '\n') - contents = [ text ] + text = text.replace(b'\x0d\x00\x0a\x00', b'\x0a\x00') + contents = [text] if type_base == "apple": # Apple filetype files will be streamed as a concatenation of @@ -2804,36 +3194,42 @@ class P4Sync(Command, P4UserMap): # Ideally, someday, this script can learn how to generate # appledouble files directly and import those to git, but # non-mac machines can never find a use for apple filetype. - print "\nIgnoring apple filetype file %s" % file['depotFile'] + print("\nIgnoring apple filetype file %s" % file['depotFile']) return + if type_base == "utf8": + # The type utf8 explicitly means utf8 *with BOM*. These are + # streamed just like regular text files, however, without + # the BOM in the stream. + # Therefore, to accurately import these files into git, we + # need to explicitly re-add the BOM before writing. + # 'contents' is a set of bytes in this case, so create the + # BOM prefix as a b'' literal. + contents = [b'\xef\xbb\xbf' + contents[0]] + contents[1:] + # Note that we do not try to de-mangle keywords on utf16 files, # even though in theory somebody may want that. - pattern = p4_keywords_regexp_for_type(type_base, type_mods) - if pattern: - regexp = re.compile(pattern, re.VERBOSE) - text = ''.join(contents) - text = regexp.sub(r'$\1$', text) - contents = [ text ] + regexp = p4_keywords_regexp_for_type(type_base, type_mods) + if regexp: + contents = [regexp.sub(br'$\1$', c) for c in contents] if self.largeFileSystem: - (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents) + git_mode, contents = self.largeFileSystem.processContent(git_mode, relPath, contents) self.writeToGitStream(git_mode, relPath, contents) def streamOneP4Deletion(self, file): - relPath = self.stripRepoPath(file['path'], self.branchPrefixes) - relPath = self.encodeWithUTF8(relPath) + relPath = self.stripRepoPath(decode_path(file['path']), self.branchPrefixes) if verbose: sys.stdout.write("delete %s\n" % relPath) sys.stdout.flush() - self.gitStream.write("D %s\n" % relPath) + self.gitStream.write(encode_text_stream(u'D {}\n'.format(relPath))) if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath): self.largeFileSystem.removeLargeFile(relPath) - # handle another chunk of streaming data def streamP4FilesCb(self, marshalled): + """Handle another chunk of streaming data.""" # catch p4 errors and complain err = None @@ -2845,9 +3241,8 @@ class P4Sync(Command, P4UserMap): if not err and 'fileSize' in self.stream_file: required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree()) if required_bytes > 0: - err = 'Not enough space left on %s! Free at least %i MB.' % ( - os.getcwd(), required_bytes/1024/1024 - ) + err = 'Not enough space left on %s! Free at least %s.' % ( + os.getcwd(), format_size_human_readable(required_bytes)) if err: f = None @@ -2866,7 +3261,7 @@ class P4Sync(Command, P4UserMap): else: die("Error from p4 print: %s" % err) - if marshalled.has_key('depotFile') and self.stream_have_file_info: + if 'depotFile' in marshalled and self.stream_have_file_info: # start of a new file - output the old one first self.streamOneP4File(self.stream_file, self.stream_contents) self.stream_file = {} @@ -2885,19 +3280,22 @@ class P4Sync(Command, P4UserMap): self.stream_file[k] = marshalled[k] if (verbose and - 'streamContentSize' in self.stream_file and - 'fileSize' in self.stream_file and - 'depotFile' in self.stream_file): + 'streamContentSize' in self.stream_file and + 'fileSize' in self.stream_file and + 'depotFile' in self.stream_file): size = int(self.stream_file["fileSize"]) if size > 0: progress = 100*self.stream_file['streamContentSize']/size - sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024))) + sys.stdout.write('\r%s %d%% (%s)' % ( + self.stream_file['depotFile'], progress, + format_size_human_readable(size))) sys.stdout.flush() self.stream_have_file_info = True - # Stream directly from "p4 files" into "git fast-import" def streamP4Files(self, files): + """Stream directly from "p4 files" into "git fast-import.""" + filesForCommit = [] filesToRead = [] filesToDelete = [] @@ -2927,9 +3325,9 @@ class P4Sync(Command, P4UserMap): if 'shelved_cl' in f: # Handle shelved CLs using the "p4 print file@=N" syntax to print # the contents - fileArg = '%s@=%d' % (f['path'], f['shelved_cl']) + fileArg = f['path'] + encode_text_stream('@={}'.format(f['shelved_cl'])) else: - fileArg = '%s#%s' % (f['path'], f['rev']) + fileArg = f['path'] + encode_text_stream('#{}'.format(f['rev'])) fileArgs.append(fileArg) @@ -2938,26 +3336,28 @@ class P4Sync(Command, P4UserMap): cb=streamP4FilesCbSelf) # do the last chunk - if self.stream_file.has_key('depotFile'): + if 'depotFile' in self.stream_file: self.streamOneP4File(self.stream_file, self.stream_contents) def make_email(self, userid): if userid in self.users: return self.users[userid] else: - return "%s <a@b>" % userid + userid_bytes = metadata_stream_to_writable_bytes(userid) + return b"%s <a@b>" % userid_bytes def streamTag(self, gitStream, labelName, labelDetails, commit, epoch): - """ Stream a p4 tag. - commit is either a git commit, or a fast-import mark, ":<p4commit>" - """ + """Stream a p4 tag. + + Commit is either a git commit, or a fast-import mark, ":<p4commit>". + """ if verbose: - print "writing tag %s for commit %s" % (labelName, commit) + print("writing tag %s for commit %s" % (labelName, commit)) gitStream.write("tag %s\n" % labelName) gitStream.write("from %s\n" % commit) - if labelDetails.has_key('Owner'): + if 'Owner' in labelDetails: owner = labelDetails["Owner"] else: owner = None @@ -2968,12 +3368,13 @@ class P4Sync(Command, P4UserMap): email = self.make_email(owner) else: email = self.make_email(self.p4UserId()) - tagger = "%s %s %s" % (email, epoch, self.tz) - gitStream.write("tagger %s\n" % tagger) + gitStream.write("tagger ") + gitStream.write(email) + gitStream.write(" %s %s\n" % (epoch, self.tz)) - print "labelDetails=",labelDetails - if labelDetails.has_key('Description'): + print("labelDetails=", labelDetails) + if 'Description' in labelDetails: description = labelDetails['Description'] else: description = 'Label from git p4' @@ -2999,7 +3400,46 @@ class P4Sync(Command, P4UserMap): print('Ignoring file outside of prefix: {0}'.format(path)) return hasPrefix - def commit(self, details, files, branch, parent = ""): + def findShadowedFiles(self, files, change): + """Perforce allows you commit files and directories with the same name, + so you could have files //depot/foo and //depot/foo/bar both checked + in. A p4 sync of a repository in this state fails. Deleting one of + the files recovers the repository. + + Git will not allow the broken state to exist and only the most + recent of the conflicting names is left in the repository. When one + of the conflicting files is deleted we need to re-add the other one + to make sure the git repository recovers in the same way as + perforce. + """ + + deleted = [f for f in files if f['action'] in self.delete_actions] + to_check = set() + for f in deleted: + path = decode_path(f['path']) + to_check.add(path + '/...') + while True: + path = path.rsplit("/", 1)[0] + if path == "/" or path in to_check: + break + to_check.add(path) + to_check = ['%s@%s' % (wildcard_encode(p), change) for p in to_check + if self.hasBranchPrefix(p)] + if to_check: + stat_result = p4CmdList(["-x", "-", "fstat", "-T", + "depotFile,headAction,headRev,headType"], stdin=to_check) + for record in stat_result: + if record['code'] != 'stat': + continue + if record['headAction'] in self.delete_actions: + continue + files.append({ + 'action': 'add', + 'path': record['depotFile'], + 'rev': record['headRev'], + 'type': record['headType']}) + + def commit(self, details, files, branch, parent="", allow_empty=False): epoch = details["time"] author = details["user"] jobs = self.extractJobsFromCommit(details) @@ -3007,13 +3447,19 @@ class P4Sync(Command, P4UserMap): if self.verbose: print('commit into {0}'.format(branch)) + files = [f for f in files + if self.hasBranchPrefix(decode_path(f['path']))] + self.findShadowedFiles(files, details['change']) + if self.clientSpecDirs: self.clientSpecDirs.update_client_spec_path_cache(files) - files = [f for f in files - if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])] + files = [f for f in files if self.inClientSpec(decode_path(f['path']))] - if not files and not gitConfigBool('git-p4.keepEmptyCommits'): + if gitConfigBool('git-p4.keepEmptyCommits'): + allow_empty = True + + if not files and not allow_empty: print('Ignoring revision {0} as it would produce an empty commit.' .format(details['change'])) return @@ -3021,12 +3467,12 @@ class P4Sync(Command, P4UserMap): self.gitStream.write("commit %s\n" % branch) self.gitStream.write("mark :%s\n" % details["change"]) self.committedChanges.add(int(details["change"])) - committer = "" if author not in self.users: self.getUserMapFromPerforceServer() - committer = "%s %s %s" % (self.make_email(author), epoch, self.tz) - self.gitStream.write("committer %s\n" % committer) + self.gitStream.write("committer ") + self.gitStream.write(self.make_email(author)) + self.gitStream.write(" %s %s\n" % (epoch, self.tz)) self.gitStream.write("data <<EOT\n") self.gitStream.write(details["desc"]) @@ -3044,7 +3490,7 @@ class P4Sync(Command, P4UserMap): if len(parent) > 0: if self.verbose: - print "parent %s" % parent + print("parent %s" % parent) self.gitStream.write("from %s\n" % parent) self.streamP4Files(files) @@ -3052,12 +3498,12 @@ class P4Sync(Command, P4UserMap): change = int(details["change"]) - if self.labels.has_key(change): + if change in self.labels: label = self.labels[change] labelDetails = label[0] labelRevisions = label[1] if self.verbose: - print "Change %s is labelled %s" % (change, labelDetails) + print("Change %s is labelled %s" % (change, labelDetails)) files = p4CmdList(["files"] + ["%s...@%s" % (p, change) for p in self.branchPrefixes]) @@ -3075,28 +3521,31 @@ class P4Sync(Command, P4UserMap): else: if not self.silent: - print ("Tag %s does not match with change %s: files do not match." + print("Tag %s does not match with change %s: files do not match." % (labelDetails["label"], change)) else: if not self.silent: - print ("Tag %s does not match with change %s: file count is different." + print("Tag %s does not match with change %s: file count is different." % (labelDetails["label"], change)) - # Build a dictionary of changelists and labels, for "detect-labels" option. def getLabels(self): + """Build a dictionary of changelists and labels, for "detect-labels" + option. + """ + self.labels = {} l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths]) if len(l) > 0 and not self.silent: - print "Finding files belonging to labels in %s" % `self.depotPaths` + print("Finding files belonging to labels in %s" % self.depotPaths) for output in l: label = output["label"] revisions = {} newestChange = 0 if self.verbose: - print "Querying files for label %s" % label + print("Querying files for label %s" % label) for file in p4CmdList(["files"] + ["%s...@%s" % (p, label) for p in self.depotPaths]): @@ -3108,15 +3557,16 @@ class P4Sync(Command, P4UserMap): self.labels[newestChange] = [output, revisions] if self.verbose: - print "Label changes: %s" % self.labels.keys() + print("Label changes: %s" % self.labels.keys()) - # Import p4 labels as git tags. A direct mapping does not - # exist, so assume that if all the files are at the same revision - # then we can use that, or it's something more complicated we should - # just ignore. def importP4Labels(self, stream, p4Labels): + """Import p4 labels as git tags. A direct mapping does not exist, so + assume that if all the files are at the same revision then we can + use that, or it's something more complicated we should just ignore. + """ + if verbose: - print "import p4 labels: " + ' '.join(p4Labels) + print("import p4 labels: " + ' '.join(p4Labels)) ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels") validLabelRegexp = gitConfig("git-p4.labelImportRegexp") @@ -3129,7 +3579,7 @@ class P4Sync(Command, P4UserMap): if not m.match(name): if verbose: - print "label %s does not match regexp %s" % (name,validLabelRegexp) + print("label %s does not match regexp %s" % (name, validLabelRegexp)) continue if name in ignoredP4Labels: @@ -3141,7 +3591,7 @@ class P4Sync(Command, P4UserMap): change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name) for p in self.depotPaths]) - if change.has_key('change'): + if 'change' in change: # find the corresponding git commit; take the oldest commit changelist = int(change['change']) if changelist in self.committedChanges: @@ -3151,7 +3601,7 @@ class P4Sync(Command, P4UserMap): gitCommit = read_pipe(["git", "rev-list", "--max-count=1", "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True) if len(gitCommit) == 0: - print "importing label %s: could not find git commit for changelist %d" % (name, changelist) + print("importing label %s: could not find git commit for changelist %d" % (name, changelist)) else: commitFound = True gitCommit = gitCommit.strip() @@ -3161,16 +3611,16 @@ class P4Sync(Command, P4UserMap): try: tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S") except ValueError: - print "Could not convert label time %s" % labelDetails['Update'] + print("Could not convert label time %s" % labelDetails['Update']) tmwhen = 1 when = int(time.mktime(tmwhen)) self.streamTag(stream, name, labelDetails, gitCommit, when) if verbose: - print "p4 label %s mapped to git commit %s" % (name, gitCommit) + print("p4 label %s mapped to git commit %s" % (name, gitCommit)) else: if verbose: - print "Label %s has no changelists - possibly deleted?" % name + print("Label %s has no changelists - possibly deleted?" % name) if not commitFound: # We can't import this label; don't try again as it will get very @@ -3185,22 +3635,19 @@ class P4Sync(Command, P4UserMap): p = p[:-1] p = p[p.strip().rfind("/") + 1:] if not p.endswith("/"): - p += "/" + p += "/" return p def getBranchMapping(self): lostAndFoundBranches = set() user = gitConfig("git-p4.branchUser") - if len(user) > 0: - command = "branches -u %s" % user - else: - command = "branches" - for info in p4CmdList(command): + for info in p4CmdList( + ["branches"] + (["-u", user] if len(user) > 0 else [])): details = p4Cmd(["branch", "-o", info["branch"]]) viewIdx = 0 - while details.has_key("View%s" % viewIdx): + while "View%s" % viewIdx in details: paths = details["View%s" % viewIdx].split(" ") viewIdx = viewIdx + 1 # require standard //depot/foo/... //depot/bar/... mapping @@ -3208,15 +3655,15 @@ class P4Sync(Command, P4UserMap): continue source = paths[0] destination = paths[1] - ## HACK + # HACK if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]): source = source[len(self.depotPaths[0]):-4] destination = destination[len(self.depotPaths[0]):-4] if destination in self.knownBranches: if not self.silent: - print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination) - print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination) + print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)) + print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)) continue self.knownBranches[destination] = source @@ -3237,7 +3684,7 @@ class P4Sync(Command, P4UserMap): configBranches = gitConfigList("git-p4.branchList") for branch in configBranches: if branch: - (source, destination) = branch.split(":") + source, destination = branch.split(":") self.knownBranches[destination] = source lostAndFoundBranches.discard(destination) @@ -3245,7 +3692,6 @@ class P4Sync(Command, P4UserMap): if source not in self.knownBranches: lostAndFoundBranches.add(source) - for branch in lostAndFoundBranches: self.knownBranches[branch] = branch @@ -3266,7 +3712,7 @@ class P4Sync(Command, P4UserMap): d["options"] = ' '.join(sorted(option_keys.keys())) def readOptions(self, d): - self.keepRepoPath = (d.has_key('options') + self.keepRepoPath = ('options' in d and ('keepRepoPath' in d['options'])) def gitRefForBranch(self, branch): @@ -3280,88 +3726,86 @@ class P4Sync(Command, P4UserMap): def gitCommitByP4Change(self, ref, change): if self.verbose: - print "looking in ref " + ref + " for change %s using bisect..." % change + print("looking in ref " + ref + " for change %s using bisect..." % change) earliestCommit = "" latestCommit = parseRevision(ref) while True: if self.verbose: - print "trying: earliest %s latest %s" % (earliestCommit, latestCommit) - next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip() + print("trying: earliest %s latest %s" % (earliestCommit, latestCommit)) + next = read_pipe(["git", "rev-list", "--bisect", + latestCommit, earliestCommit]).strip() if len(next) == 0: if self.verbose: - print "argh" + print("argh") return "" log = extractLogMessageFromGitCommit(next) settings = extractSettingsGitLog(log) currentChange = int(settings['change']) if self.verbose: - print "current change %s" % currentChange + print("current change %s" % currentChange) if currentChange == change: if self.verbose: - print "found %s" % next + print("found %s" % next) return next if currentChange < change: earliestCommit = "^%s" % next else: - latestCommit = "%s" % next + if next == latestCommit: + die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref, change)) + latestCommit = "%s^@" % next return "" def importNewBranch(self, branch, maxChange): # make fast-import flush all changes to disk and update the refs using the checkpoint # command so that we can try to find the branch parent in the git history - self.gitStream.write("checkpoint\n\n"); - self.gitStream.flush(); + self.gitStream.write("checkpoint\n\n") + self.gitStream.flush() branchPrefix = self.depotPaths[0] + branch + "/" range = "@1,%s" % maxChange - #print "prefix" + branchPrefix changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size) if len(changes) <= 0: return False firstChange = changes[0] - #print "first change in branch: %s" % firstChange sourceBranch = self.knownBranches[branch] sourceDepotPath = self.depotPaths[0] + sourceBranch sourceRef = self.gitRefForBranch(sourceBranch) - #print "source " + sourceBranch branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"]) - #print "branch parent: %s" % branchParentChange gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange) if len(gitParent) > 0: self.initialParents[self.gitRefForBranch(branch)] = gitParent - #print "parent git commit: %s" % gitParent self.importChanges(changes) return True def searchParent(self, parent, branch, target): - parentFound = False - for blob in read_pipe_lines(["git", "rev-list", "--reverse", + targetTree = read_pipe(["git", "rev-parse", + "{}^{{tree}}".format(target)]).strip() + for line in read_pipe_lines(["git", "rev-list", "--format=%H %T", "--no-merges", parent]): - blob = blob.strip() - if len(read_pipe(["git", "diff-tree", blob, target])) == 0: - parentFound = True + if line.startswith("commit "): + continue + commit, tree = line.strip().split(" ") + if tree == targetTree: if self.verbose: - print "Found parent of %s in commit %s" % (branch, blob) - break - if parentFound: - return blob - else: - return None + print("Found parent of %s in commit %s" % (branch, commit)) + return commit + return None - def importChanges(self, changes, shelved=False, origin_revision=0): + def importChanges(self, changes, origin_revision=0): cnt = 1 for change in changes: - description = p4_describe(change, shelved) + description = p4_describe(change) self.updateOptionDict(description) if not self.silent: - sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) + sys.stdout.write("\rImporting revision %s (%d%%)" % ( + change, (cnt * 100) // len(changes))) sys.stdout.flush() cnt = cnt + 1 @@ -3369,16 +3813,16 @@ class P4Sync(Command, P4UserMap): if self.detectBranches: branches = self.splitFilesIntoBranches(description) for branch in branches.keys(): - ## HACK --hwn + # HACK --hwn branchPrefix = self.depotPaths[0] + branch + "/" - self.branchPrefixes = [ branchPrefix ] + self.branchPrefixes = [branchPrefix] parent = "" filesForCommit = branches[branch] if self.verbose: - print "branch is %s" % branch + print("branch is %s" % branch) self.updatedBranches.add(branch) @@ -3391,21 +3835,21 @@ class P4Sync(Command, P4UserMap): fullBranch = self.projectName + branch if fullBranch not in self.p4BranchesInGit: if not self.silent: - print("\n Importing new branch %s" % fullBranch); + print("\n Importing new branch %s" % fullBranch) if self.importNewBranch(branch, change - 1): parent = "" self.p4BranchesInGit.append(fullBranch) if not self.silent: - print("\n Resuming with change %s" % change); + print("\n Resuming with change %s" % change) if self.verbose: - print "parent determined through known branches: %s" % parent + print("parent determined through known branches: %s" % parent) branch = self.gitRefForBranch(branch) parent = self.gitRefForBranch(parent) if self.verbose: - print "looking for initial parent for %s; current parent is %s" % (branch, parent) + print("looking for initial parent for %s; current parent is %s" % (branch, parent)) if len(parent) == 0 and branch in self.initialParents: parent = self.initialParents[branch] @@ -3415,7 +3859,7 @@ class P4Sync(Command, P4UserMap): if len(parent) > 0: tempBranch = "%s/%d" % (self.tempBranchLocation, change) if self.verbose: - print "Creating temporary branch: " + tempBranch + print("Creating temporary branch: " + tempBranch) self.commit(description, filesForCommit, tempBranch) self.tempBranches.append(tempBranch) self.checkpoint() @@ -3424,16 +3868,16 @@ class P4Sync(Command, P4UserMap): self.commit(description, filesForCommit, branch, blob) else: if self.verbose: - print "Parent of %s not found. Committing into head of %s" % (branch, parent) + print("Parent of %s not found. Committing into head of %s" % (branch, parent)) self.commit(description, filesForCommit, branch, parent) else: - files = self.extractFilesFromCommit(description, shelved, change, origin_revision) + files = self.extractFilesFromCommit(description) self.commit(description, files, self.branch, self.initialParent) # only needed once, to connect to the previous commit self.initialParent = "" except IOError: - print self.gitError.read() + print(self.gitError.read()) sys.exit(1) def sync_origin_only(self): @@ -3441,11 +3885,11 @@ class P4Sync(Command, P4UserMap): self.hasOrigin = originP4BranchesExist() if self.hasOrigin: if not self.silent: - print 'Syncing with origin first, using "git fetch origin"' - system("git fetch origin") + print('Syncing with origin first, using "git fetch origin"') + system(["git", "fetch", "origin"]) def importHeadRevision(self, revision): - print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch) + print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)) details = {} details["user"] = "git perforce import user" @@ -3455,7 +3899,7 @@ class P4Sync(Command, P4UserMap): newestRevision = 0 fileCnt = 0 - fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths] + fileArgs = ["%s...%s" % (p, revision) for p in self.depotPaths] for info in p4CmdList(["files"] + fileArgs): @@ -3464,24 +3908,21 @@ class P4Sync(Command, P4UserMap): % info['data']) if info['data'].find("must refer to client") >= 0: sys.stderr.write("This particular p4 error is misleading.\n") - sys.stderr.write("Perhaps the depot path was misspelled.\n"); + sys.stderr.write("Perhaps the depot path was misspelled.\n") sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths)) sys.exit(1) if 'p4ExitCode' in info: sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode']) sys.exit(1) - change = int(info["change"]) if change > newestRevision: newestRevision = change if info["action"] in self.delete_actions: - # don't increase the file cnt, otherwise details["depotFile123"] will have gaps! - #fileCnt = fileCnt + 1 continue - for prop in ["depotFile", "rev", "action", "type" ]: + for prop in ["depotFile", "rev", "action", "type"]: details["%s%s" % (prop, fileCnt)] = info[prop] fileCnt = fileCnt + 1 @@ -3496,25 +3937,105 @@ class P4Sync(Command, P4UserMap): self.updateOptionDict(details) try: self.commit(details, self.extractFilesFromCommit(details), self.branch) - except IOError: - print "IO error with git fast-import. Is your git version recent enough?" - print self.gitError.read() + except IOError as err: + print("IO error with git fast-import. Is your git version recent enough?") + print("IO error details: {}".format(err)) + print(self.gitError.read()) + + def importRevisions(self, args, branch_arg_given): + changes = [] + + if len(self.changesFile) > 0: + with open(self.changesFile) as f: + output = f.readlines() + changeSet = set() + for line in output: + changeSet.add(int(line)) + + for change in changeSet: + changes.append(change) + + changes.sort() + else: + # catch "git p4 sync" with no new branches, in a repo that + # does not have any existing p4 branches + if len(args) == 0: + if not self.p4BranchesInGit: + raise P4CommandException("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.") + + # The default branch is master, unless --branch is used to + # specify something else. Make sure it exists, or complain + # nicely about how to use --branch. + if not self.detectBranches: + if not branch_exists(self.branch): + if branch_arg_given: + raise P4CommandException("Error: branch %s does not exist." % self.branch) + else: + raise P4CommandException("Error: no branch %s; perhaps specify one with --branch." % + self.branch) + + if self.verbose: + print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths), + self.changeRange)) + changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size) + + if len(self.maxChanges) > 0: + changes = changes[:min(int(self.maxChanges), len(changes))] + + if len(changes) == 0: + if not self.silent: + print("No changes to import!") + else: + if not self.silent and not self.detectBranches: + print("Import destination: %s" % self.branch) + + self.updatedBranches = set() + + if not self.detectBranches: + if args: + # start a new branch + self.initialParent = "" + else: + # build on a previous revision + self.initialParent = parseRevision(self.branch) + + self.importChanges(changes) + + if not self.silent: + print("") + if len(self.updatedBranches) > 0: + sys.stdout.write("Updated branches: ") + for b in self.updatedBranches: + sys.stdout.write("%s " % b) + sys.stdout.write("\n") def openStreams(self): self.importProcess = subprocess.Popen(["git", "fast-import"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE); + stderr=subprocess.PIPE) self.gitOutput = self.importProcess.stdout self.gitStream = self.importProcess.stdin self.gitError = self.importProcess.stderr + if bytes is not str: + # Wrap gitStream.write() so that it can be called using `str` arguments + def make_encoded_write(write): + def encoded_write(s): + return write(s.encode() if isinstance(s, str) else s) + return encoded_write + + self.gitStream.write = make_encoded_write(self.gitStream.write) + def closeStreams(self): + if self.gitStream is None: + return self.gitStream.close() if self.importProcess.wait() != 0: die("fast-import failed: %s" % self.gitError.read()) self.gitOutput.close() self.gitError.close() + self.gitStream = None def run(self, args): if self.importIntoRemotes: @@ -3528,8 +4049,8 @@ class P4Sync(Command, P4UserMap): if len(self.branch) == 0: self.branch = self.refPrefix + "master" if gitBranchExists("refs/heads/p4") and self.importIntoRemotes: - system("git update-ref %s refs/heads/p4" % self.branch) - system("git branch -D p4") + system(["git", "update-ref", self.branch, "refs/heads/p4"]) + system(["git", "branch", "-D", "p4"]) # accept either the command-line option, or the configuration variable if self.useClientSpec: @@ -3552,32 +4073,36 @@ class P4Sync(Command, P4UserMap): # restrict to just this one, disabling detect-branches if branch_arg_given: - short = self.branch.split("/")[-1] + short = shortP4Ref(self.branch, self.importIntoRemotes) if short in branches: - self.p4BranchesInGit = [ short ] + self.p4BranchesInGit = [short] + elif self.branch.startswith('refs/') and \ + branchExists(self.branch) and \ + '[git-p4:' in extractLogMessageFromGitCommit(self.branch): + self.p4BranchesInGit = [self.branch] else: self.p4BranchesInGit = branches.keys() if len(self.p4BranchesInGit) > 1: if not self.silent: - print "Importing from/into multiple branches" + print("Importing from/into multiple branches") self.detectBranches = True for branch in branches.keys(): self.initialParents[self.refPrefix + branch] = \ branches[branch] if self.verbose: - print "branches: %s" % self.p4BranchesInGit + print("branches: %s" % self.p4BranchesInGit) p4Change = 0 for branch in self.p4BranchesInGit: - logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch) + logMsg = extractLogMessageFromGitCommit(fullP4Ref(branch, + self.importIntoRemotes)) settings = extractSettingsGitLog(logMsg) self.readOptions(settings) - if (settings.has_key('depot-paths') - and settings.has_key ('change')): + if 'depot-paths' in settings and 'change' in settings: change = int(settings['change']) + 1 p4Change = max(p4Change, change) @@ -3590,11 +4115,11 @@ class P4Sync(Command, P4UserMap): prev_list = prev.split("/") cur_list = cur.split("/") for i in range(0, min(len(cur_list), len(prev_list))): - if cur_list[i] <> prev_list[i]: + if cur_list[i] != prev_list[i]: i = i - 1 break - paths.append ("/".join(cur_list[:i + 1])) + paths.append("/".join(cur_list[:i + 1])) self.previousDepotPaths = paths @@ -3602,29 +4127,18 @@ class P4Sync(Command, P4UserMap): self.depotPaths = sorted(self.previousDepotPaths) self.changeRange = "@%s,#head" % p4Change if not self.silent and not self.detectBranches: - print "Performing incremental import into %s git branch" % self.branch - - # accept multiple ref name abbreviations: - # refs/foo/bar/branch -> use it exactly - # p4/branch -> prepend refs/remotes/ or refs/heads/ - # branch -> prepend refs/remotes/p4/ or refs/heads/p4/ - if not self.branch.startswith("refs/"): - if self.importIntoRemotes: - prepend = "refs/remotes/" - else: - prepend = "refs/heads/" - if not self.branch.startswith("p4/"): - prepend += "p4/" - self.branch = prepend + self.branch + print("Performing incremental import into %s git branch" % self.branch) + + self.branch = fullP4Ref(self.branch, self.importIntoRemotes) if len(args) == 0 and self.depotPaths: if not self.silent: - print "Depot paths: %s" % ' '.join(self.depotPaths) + print("Depot paths: %s" % ' '.join(self.depotPaths)) else: if self.depotPaths and self.depotPaths != args: - print ("previous import used depot path %s and now %s was specified. " - "This doesn't work!" % (' '.join (self.depotPaths), - ' '.join (args))) + print("previous import used depot path %s and now %s was specified. " + "This doesn't work!" % (' '.join(self.depotPaths), + ' '.join(args))) sys.exit(1) self.depotPaths = sorted(args) @@ -3664,7 +4178,7 @@ class P4Sync(Command, P4UserMap): if len(self.changesFile) == 0: revision = "#head" - p = re.sub ("\.\.\.$", "", p) + p = re.sub("\.\.\.$", "", p) if not p.endswith("/"): p += "/" @@ -3678,10 +4192,10 @@ class P4Sync(Command, P4UserMap): self.loadUserMapFromCache() self.labels = {} if self.detectLabels: - self.getLabels(); + self.getLabels() if self.detectBranches: - ## FIXME - what's a P4 projectName ? + # FIXME - what's a P4 projectName ? self.projectName = self.guessProjectName() if self.hasOrigin: @@ -3689,101 +4203,50 @@ class P4Sync(Command, P4UserMap): else: self.getBranchMapping() if self.verbose: - print "p4-git branches: %s" % self.p4BranchesInGit - print "initial parents: %s" % self.initialParents + print("p4-git branches: %s" % self.p4BranchesInGit) + print("initial parents: %s" % self.initialParents) for b in self.p4BranchesInGit: if b != "master": - ## FIXME + # FIXME b = b[len(self.projectName):] self.createdBranches.add(b) - self.openStreams() - - if revision: - self.importHeadRevision(revision) - else: - changes = [] - - if len(self.changesFile) > 0: - output = open(self.changesFile).readlines() - changeSet = set() - for line in output: - changeSet.add(int(line)) - - for change in changeSet: - changes.append(change) - - changes.sort() - else: - # catch "git p4 sync" with no new branches, in a repo that - # does not have any existing p4 branches - if len(args) == 0: - if not self.p4BranchesInGit: - die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.") - - # The default branch is master, unless --branch is used to - # specify something else. Make sure it exists, or complain - # nicely about how to use --branch. - if not self.detectBranches: - if not branch_exists(self.branch): - if branch_arg_given: - die("Error: branch %s does not exist." % self.branch) - else: - die("Error: no branch %s; perhaps specify one with --branch." % - self.branch) + p4_check_access() - if self.verbose: - print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths), - self.changeRange) - changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size) + self.openStreams() - if len(self.maxChanges) > 0: - changes = changes[:min(int(self.maxChanges), len(changes))] + err = None - if len(changes) == 0: - if not self.silent: - print "No changes to import!" + try: + if revision: + self.importHeadRevision(revision) else: - if not self.silent and not self.detectBranches: - print "Import destination: %s" % self.branch + self.importRevisions(args, branch_arg_given) - self.updatedBranches = set() + if gitConfigBool("git-p4.importLabels"): + self.importLabels = True - if not self.detectBranches: - if args: - # start a new branch - self.initialParent = "" - else: - # build on a previous revision - self.initialParent = parseRevision(self.branch) + if self.importLabels: + p4Labels = getP4Labels(self.depotPaths) + gitTags = getGitTags() - self.importChanges(changes) + missingP4Labels = p4Labels - gitTags + self.importP4Labels(self.gitStream, missingP4Labels) - if not self.silent: - print "" - if len(self.updatedBranches) > 0: - sys.stdout.write("Updated branches: ") - for b in self.updatedBranches: - sys.stdout.write("%s " % b) - sys.stdout.write("\n") - - if gitConfigBool("git-p4.importLabels"): - self.importLabels = True - - if self.importLabels: - p4Labels = getP4Labels(self.depotPaths) - gitTags = getGitTags() + except P4CommandException as e: + err = e - missingP4Labels = p4Labels - gitTags - self.importP4Labels(self.gitStream, missingP4Labels) + finally: + self.closeStreams() - self.closeStreams() + if err: + die(str(err)) # Cleanup temporary branches created during import if self.tempBranches != []: for branch in self.tempBranches: - read_pipe("git update-ref -d %s" % branch) + read_pipe(["git", "update-ref", "-d", branch]) os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation)) # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow @@ -3795,6 +4258,7 @@ class P4Sync(Command, P4UserMap): return True + class P4Rebase(Command): def __init__(self): Command.__init__(self) @@ -3814,23 +4278,25 @@ class P4Rebase(Command): def rebase(self): if os.system("git update-index --refresh") != 0: - die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash."); - if len(read_pipe("git diff-index HEAD --")) > 0: - die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash."); + die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.") + if len(read_pipe(["git", "diff-index", "HEAD", "--"])) > 0: + die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.") - [upstream, settings] = findUpstreamBranchPoint() + upstream, settings = findUpstreamBranchPoint() if len(upstream) == 0: die("Cannot find upstream branchpoint for rebase") # the branchpoint may be p4/foo~3, so strip off the parent upstream = re.sub("~[0-9]+$", "", upstream) - print "Rebasing the current branch onto %s" % upstream - oldHead = read_pipe("git rev-parse HEAD").strip() - system("git rebase %s" % upstream) - system("git diff-tree --stat --summary -M %s HEAD --" % oldHead) + print("Rebasing the current branch onto %s" % upstream) + oldHead = read_pipe(["git", "rev-parse", "HEAD"]).strip() + system(["git", "rebase", upstream]) + system(["git", "diff-tree", "--stat", "--summary", "-M", oldHead, + "HEAD", "--"]) return True + class P4Clone(P4Sync): def __init__(self): P4Sync.__init__(self) @@ -3848,7 +4314,7 @@ class P4Clone(P4Sync): self.cloneBare = False def defaultDestination(self, args): - ## TODO: use common prefix of args? + # TODO: use common prefix of args? depotPath = args[0] depotDir = re.sub("(@[^@]*)$", "", depotPath) depotDir = re.sub("(#[^#]*)$", "", depotDir) @@ -3870,7 +4336,6 @@ class P4Clone(P4Sync): self.cloneDestination = depotPaths[-1] depotPaths = depotPaths[:-1] - self.cloneExclude = ["/"+p for p in self.cloneExclude] for p in depotPaths: if not p.startswith("//"): sys.stderr.write('Depot paths must start with "//": %s\n' % p) @@ -3879,37 +4344,46 @@ class P4Clone(P4Sync): if not self.cloneDestination: self.cloneDestination = self.defaultDestination(args) - print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination) + print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)) if not os.path.exists(self.cloneDestination): os.makedirs(self.cloneDestination) chdir(self.cloneDestination) - init_cmd = [ "git", "init" ] + init_cmd = ["git", "init"] if self.cloneBare: init_cmd.append("--bare") retcode = subprocess.call(init_cmd) if retcode: - raise CalledProcessError(retcode, init_cmd) + raise subprocess.CalledProcessError(retcode, init_cmd) if not P4Sync.run(self, depotPaths): return False # create a master branch and check out a work tree if gitBranchExists(self.branch): - system([ "git", "branch", "master", self.branch ]) + system(["git", "branch", currentGitBranch(), self.branch]) if not self.cloneBare: - system([ "git", "checkout", "-f" ]) + system(["git", "checkout", "-f"]) else: - print 'Not checking out any branch, use ' \ - '"git checkout -q -b master <branch>"' + print('Not checking out any branch, use ' + '"git checkout -q -b master <branch>"') # auto-set this variable if invoked with --use-client-spec if self.useClientSpec_from_options: - system("git config --bool git-p4.useclientspec true") + system(["git", "config", "--bool", "git-p4.useclientspec", "true"]) + + # persist any git-p4 encoding-handling config options passed in for clone: + if gitConfig('git-p4.metadataDecodingStrategy'): + system(["git", "config", "git-p4.metadataDecodingStrategy", gitConfig('git-p4.metadataDecodingStrategy')]) + if gitConfig('git-p4.metadataFallbackEncoding'): + system(["git", "config", "git-p4.metadataFallbackEncoding", gitConfig('git-p4.metadataFallbackEncoding')]) + if gitConfig('git-p4.pathEncoding'): + system(["git", "config", "git-p4.pathEncoding", gitConfig('git-p4.pathEncoding')]) return True + class P4Unshelve(Command): def __init__(self): Command.__init__(self) @@ -3923,38 +4397,62 @@ class P4Unshelve(Command): ] self.verbose = False self.noCommit = False - self.destbranch = "refs/remotes/p4/unshelved" + self.destbranch = "refs/remotes/p4-unshelved" def renameBranch(self, branch_name): - """ Rename the existing branch to branch_name.N - """ + """Rename the existing branch to branch_name.N .""" - found = True - for i in range(0,1000): + for i in range(0, 1000): backup_branch_name = "{0}.{1}".format(branch_name, i) if not gitBranchExists(backup_branch_name): - gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup + # Copy ref to backup + gitUpdateRef(backup_branch_name, branch_name) gitDeleteRef(branch_name) - found = True print("renamed old unshelve branch to {0}".format(backup_branch_name)) break - - if not found: - sys.exit("gave up trying to rename existing branch {0}".format(sync.branch)) + else: + sys.exit("gave up trying to rename existing branch {0}".format(branch_name)) def findLastP4Revision(self, starting_point): - """ Look back from starting_point for the first commit created by git-p4 - to find the P4 commit we are based on, and the depot-paths. - """ + """Look back from starting_point for the first commit created by git-p4 + to find the P4 commit we are based on, and the depot-paths. + """ for parent in (range(65535)): - log = extractLogMessageFromGitCommit("{0}^{1}".format(starting_point, parent)) + log = extractLogMessageFromGitCommit("{0}~{1}".format(starting_point, parent)) settings = extractSettingsGitLog(log) - if settings.has_key('change'): + if 'change' in settings: return settings sys.exit("could not find git-p4 commits in {0}".format(self.origin)) + def createShelveParent(self, change, branch_name, sync, origin): + """Create a commit matching the parent of the shelved changelist + 'change'. + """ + parent_description = p4_describe(change, shelved=True) + parent_description['desc'] = 'parent for shelved changelist {}\n'.format(change) + files = sync.extractFilesFromCommit(parent_description, shelved=False, shelved_cl=change) + + parent_files = [] + for f in files: + # if it was added in the shelved changelist, it won't exist in the parent + if f['action'] in self.add_actions: + continue + + # if it was deleted in the shelved changelist it must not be deleted + # in the parent - we might even need to create it if the origin branch + # does not have it + if f['action'] in self.delete_actions: + f['action'] = 'add' + + parent_files.append(f) + + sync.commit(parent_description, parent_files, branch_name, + parent=origin, allow_empty=True) + print("created parent commit for {0} based on {1} in {2}".format( + change, self.origin, branch_name)) + def run(self, args): if len(args) != 1: return False @@ -3964,9 +4462,8 @@ class P4Unshelve(Command): sync = P4Sync() changes = args - sync.initialParent = self.origin - # use the first change in the list to construct the branch to unshelve into + # only one change at a time change = changes[0] # if the target branch already exists, rename it @@ -3979,24 +4476,32 @@ class P4Unshelve(Command): sync.suppress_meta_comment = True settings = self.findLastP4Revision(self.origin) - origin_revision = settings['change'] sync.depotPaths = settings['depot-paths'] sync.branchPrefixes = sync.depotPaths sync.openStreams() sync.loadUserMapFromCache() sync.silent = True - sync.importChanges(changes, shelved=True, origin_revision=origin_revision) + + # create a commit for the parent of the shelved changelist + self.createShelveParent(change, branch_name, sync, self.origin) + + # create the commit for the shelved changelist itself + description = p4_describe(change, True) + files = sync.extractFilesFromCommit(description, True, change) + + sync.commit(description, files, branch_name, "") sync.closeStreams() print("unshelved changelist {0} into {1}".format(change, branch_name)) return True + class P4Branches(Command): def __init__(self): Command.__init__(self) - self.options = [ ] + self.options = [] self.description = ("Shows the git branches that hold imports and their " + "corresponding perforce depot paths") self.verbose = False @@ -4005,10 +4510,7 @@ class P4Branches(Command): if originP4BranchesExist(): createOrUpdateBranchesFromOrigin() - cmdline = "git rev-parse --symbolic " - cmdline += " --remotes" - - for line in read_pipe_lines(cmdline): + for line in read_pipe_lines(["git", "rev-parse", "--symbolic", "--remotes"]): line = line.strip() if not line.startswith('p4/') or line == "p4/HEAD": @@ -4018,9 +4520,10 @@ class P4Branches(Command): log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch) settings = extractSettingsGitLog(log) - print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]) + print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])) return True + class HelpFormatter(optparse.IndentedHelpFormatter): def __init__(self): optparse.IndentedHelpFormatter.__init__(self) @@ -4031,24 +4534,24 @@ class HelpFormatter(optparse.IndentedHelpFormatter): else: return "" + def printUsage(commands): - print "usage: %s <command> [options]" % sys.argv[0] - print "" - print "valid commands: %s" % ", ".join(commands) - print "" - print "Try %s <command> --help for command specific help." % sys.argv[0] - print "" + print("usage: %s <command> [options]" % sys.argv[0]) + print("") + print("valid commands: %s" % ", ".join(commands)) + print("") + print("Try %s <command> --help for command specific help." % sys.argv[0]) + print("") + commands = { - "debug" : P4Debug, - "submit" : P4Submit, - "commit" : P4Submit, - "sync" : P4Sync, - "rebase" : P4Rebase, - "clone" : P4Clone, - "rollback" : P4RollBack, - "branches" : P4Branches, - "unshelve" : P4Unshelve, + "submit": P4Submit, + "commit": P4Submit, + "sync": P4Sync, + "rebase": P4Rebase, + "clone": P4Clone, + "branches": P4Branches, + "unshelve": P4Unshelve, } @@ -4062,8 +4565,8 @@ def main(): klass = commands[cmdName] cmd = klass() except KeyError: - print "unknown command %s" % cmdName - print "" + print("unknown command %s" % cmdName) + print("") printUsage(commands.keys()) sys.exit(2) @@ -4078,22 +4581,27 @@ def main(): parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName), options, - description = cmd.description, - formatter = HelpFormatter()) + description=cmd.description, + formatter=HelpFormatter()) + + try: + cmd, args = parser.parse_args(sys.argv[2:], cmd) + except: + parser.print_help() + raise - (cmd, args) = parser.parse_args(sys.argv[2:], cmd); global verbose verbose = cmd.verbose if cmd.needsGit: - if cmd.gitdir == None: + if cmd.gitdir is None: cmd.gitdir = os.path.abspath(".git") if not isValidGitDir(cmd.gitdir): # "rev-parse --git-dir" without arguments will try $PWD/.git - cmd.gitdir = read_pipe("git rev-parse --git-dir").strip() + cmd.gitdir = read_pipe(["git", "rev-parse", "--git-dir"]).strip() if os.path.exists(cmd.gitdir): - cdup = read_pipe("git rev-parse --show-cdup").strip() + cdup = read_pipe(["git", "rev-parse", "--show-cdup"]).strip() if len(cdup) > 0: - chdir(cdup); + chdir(cdup) if not isValidGitDir(cmd.gitdir): if isValidGitDir(cmd.gitdir + "/.git"): |