From 55aa5714afc1310fd94936a8cd922e868d694bd2 Mon Sep 17 00:00:00 2001 From: Pete Wyckoff Date: Sat, 17 Sep 2011 19:16:14 -0400 Subject: git-p4: handle utf16 filetype properly One of the filetypes that p4 supports is utf16. Its behavior is odd in this case. The data delivered through "p4 -G print" is not encoded in utf16, although "p4 print -o" will produce the proper utf16-encoded file. When dealing with this filetype, discard the data from -G, and instead read the contents directly. An alternate approach would be to try to encode the data in python. That worked for true utf16 files, but for other files marked as utf16, p4 delivers mangled text in no recognizable encoding. Add a test case to check utf16 handling, and +k and +ko handling. Reported-by: Chris Li Acked-by: Luke Diamand Signed-off-by: Pete Wyckoff Signed-off-by: Junio C Hamano --- contrib/fast-import/git-p4 | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'contrib/fast-import') diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 2f7b27056..e69caf368 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1238,6 +1238,15 @@ class P4Sync(Command, P4UserMap): data = ''.join(contents) contents = [data[:-1]] + if file['type'].startswith("utf16"): + # p4 delivers different text in the python output to -G + # than it does when using "print -o", or normal p4 client + # operations. utf16 is converted to ascii or utf8, perhaps. + # But ascii text saved as -t utf16 is completely mangled. + # Invoke print -o to get the real contents. + text = p4_read_pipe('print -q -o - "%s"' % file['depotFile']) + contents = [ text ] + if self.isWindows and file["type"].endswith("text"): mangled = [] for data in contents: @@ -1245,6 +1254,8 @@ class P4Sync(Command, P4UserMap): mangled.append(data) contents = mangled + # Note that we do not try to de-mangle keywords on utf16 files, + # even though in theory somebody may want that. if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'): contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents) elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'): -- cgit v1.2.1