fetch-binaries.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278
  1. #!/usr/bin/env python
  2. # Sample CI usage: scripts\common\fetch-binaries.py -p windows -b release -t abc123 -n 33
  3. # Sample desktop usage: scripts\common\fetch-binaries.py -i ..\plex-dependency-builder\output\Packages\pms-depends-windows-i386-debug-dev.bz2
  4. import hashlib
  5. import optparse
  6. import ConfigParser
  7. import os, re
  8. import platform
  9. import subprocess
  10. import sys
  11. import shutil
  12. import urllib2
  13. import base64
  14. import glob
  15. # Edit these to set a new default dependencies build
  16. default_tag = "auto"
  17. default_release_build_number = "109"
  18. default_release_dir = "plexmediaplayer-dependencies-codecs"
  19. default_branch = "master"
  20. def sha1_for_file(path):
  21. hash=hashlib.sha1()
  22. fp=file(path, "rb")
  23. while True:
  24. data=fp.read(4096)
  25. if not data:
  26. break
  27. hash.update(data)
  28. return hash.hexdigest()
  29. def exec_cmd(args, env={}, supress_output=False):
  30. """ main exec_cmd function """
  31. # forward SSH_AUTH_SOCK, so that ssh-agent works
  32. if os.name != "nt" and "SSH_AUTH_SOCK" in os.environ:
  33. env = os.environ
  34. extra_env={"SSH_AUTH_SOCK":os.environ["SSH_AUTH_SOCK"]}
  35. env.update(extra_env)
  36. else:
  37. env = os.environ
  38. cmd = subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, env = env)
  39. output = ''
  40. while True:
  41. out = cmd.stdout.read(1)
  42. if out == '' and cmd.poll() != None:
  43. break
  44. if out != '':
  45. if not supress_output:
  46. sys.stdout.write(out)
  47. output += out
  48. if cmd.wait() != 0:
  49. raise Exception("Command failed: \"%s\"" % " ".join(args), output)
  50. return output
  51. platform_map={"linux-synology-i386":"synology-i686",
  52. "linux-readynas-arm":"ubuntu-arm",
  53. "linux-debian-4-i386":"debian-i686",
  54. "linux-control4-arm":"control4-arm",
  55. "linux-apm-ppc":"apm-ppc",
  56. "linux-armada-arm7":"armada-arm7",
  57. "linux-synology-arm":"synology-arm"}
  58. def platform_str():
  59. if "BUILD_TAG" in os.environ:
  60. for (k, v) in platform_map.iteritems():
  61. if k in os.environ["BUILD_TAG"]:
  62. return "linux-"+v
  63. return "linux-%s-%s"%(platform.linux_distribution()[0].strip().lower(), platform.machine())
  64. def merge_directories(src, dest, move = False):
  65. for src_dir, dirs, files in os.walk(src):
  66. dst_dir = src_dir.replace(src, dest)
  67. if not os.path.exists(dst_dir):
  68. os.mkdir(dst_dir)
  69. for file_ in files:
  70. src_file = os.path.join(src_dir, file_)
  71. dst_file = os.path.join(dst_dir, file_)
  72. if os.path.exists(dst_file):
  73. os.remove(dst_file)
  74. if move:
  75. shutil.move(src_file, dst_dir)
  76. else:
  77. shutil.copy(src_file, dst_dir)
  78. if move and os.path.exists(src):
  79. shutil.rmtree(src)
  80. def unpack_and_install(download, inputfile, installed_filepath):
  81. # Make paths absolute before changing directories
  82. inputfile = os.path.abspath(inputfile)
  83. if not os.path.exists(inputfile):
  84. print "Input file %s does not exist" % inputfile
  85. sys.exit(1)
  86. shafile = "%s.sha.txt" % inputfile
  87. installed_filepath = os.path.abspath(installed_filepath)
  88. # Go to directory.
  89. old_cwd = os.getcwd()
  90. os.chdir(options.output)
  91. # Check the SHA
  92. if options.nochecksha:
  93. print "-- Skipping SHA verification"
  94. elif os.path.exists(shafile):
  95. f = open(shafile, "r")
  96. sha = f.readline().strip()
  97. computed = sha1_for_file(inputfile)
  98. if not computed == sha:
  99. print "-- SHA didn't match: %s != %s" % (sha, computed)
  100. sys.exit(1)
  101. else:
  102. print "-- SHA %s matches" % sha
  103. f.close()
  104. else:
  105. print "-- ERROR - No SHA file is available to verify the file's integrity"
  106. sys.exit(1)
  107. # Untar the package file
  108. inputfile_tarfriendly = inputfile
  109. if os.name == "nt":
  110. pattern = re.compile(r'([a-z]):\\', re.IGNORECASE)
  111. inputfile_tarfriendly = pattern.sub('/\\1/', inputfile).replace('\\','/')
  112. # The final destination directory is the filename without a version number
  113. # The version number is the last element in the filename (by convention)
  114. packagename = os.path.splitext(os.path.basename(inputfile))[0]
  115. packagename_elements = packagename.split("-")
  116. del packagename_elements[-1]
  117. packagename_noversion = "-".join(packagename_elements)
  118. if os.path.exists(packagename_noversion):
  119. shutil.rmtree(packagename_noversion, ignore_errors=True)
  120. os.makedirs(packagename_noversion)
  121. print "-- Unpacking %s... to %s" % (os.path.basename(inputfile), packagename_noversion)
  122. if os.name == "nt":
  123. # Touch files from the package, which often arrive from the future when freshly built on our Windows build machine
  124. exec_cmd(["tar", "xjf", inputfile_tarfriendly, "-C", packagename_noversion, "--touch", "--strip-components", "1", "--no-same-owner"])
  125. else:
  126. exec_cmd(["tar", "xjf", inputfile_tarfriendly, "-C", packagename_noversion, "--strip-components", "1", "--no-same-owner"])
  127. if download and installed_filepath:
  128. # Create the installed stamp file to note our success
  129. open(installed_filepath, "wb")
  130. # Restore directory.
  131. os.chdir(old_cwd)
  132. return packagename
  133. if __name__=='__main__':
  134. parser=optparse.OptionParser()
  135. parser.add_option("-p", "--platform", action="store", type="string",
  136. dest="platform", help="Platform identifier (e.g. windows-i386)", default=None)
  137. parser.add_option("-b", "--buildconfig", action="store", type="string",
  138. dest="buildconfig", help="Build configuration (release or debug). Default is release", default="release")
  139. parser.add_option("-t", "--tag", action="store", type="string",
  140. dest="tag", help="Build tag. Default is %s" % default_tag, default=default_tag)
  141. parser.add_option("-n", "--buildnumber", action="store", type="string",
  142. dest="buildnumber", help="Build number. Default is %s for release builds" % (default_release_build_number), default=None)
  143. parser.add_option("-d", "--dir", action="store", type="string",
  144. dest="dir", help="CI build dir. Default is %s for release builds" % (default_release_dir), default=None)
  145. parser.add_option("-i", "--inputfile", action="store", type="string",
  146. dest="inputfile", help="Dependencies package filename, can be supplied instead of platform, buildconfig, tag and buildnumber", default=None)
  147. parser.add_option("-o", "--output", action="store", type="string",
  148. dest="output", help="Output directory. Default is Dependencies",
  149. default="Dependencies")
  150. parser.add_option("-x", "--nochecksha", action="store_true",
  151. dest="nochecksha", help="Don't check the SHA. Default is false",
  152. default=False)
  153. parser.add_option("-r", "--branch", action="store", type="string",
  154. dest="branch", help="Git branch", default=None)
  155. (options, args)=parser.parse_args(sys.argv)
  156. if not os.path.exists(options.output):
  157. os.makedirs(options.output)
  158. # Fail early if platform is not known
  159. download = not options.inputfile
  160. if download and not options.platform:
  161. print "ERROR - A platform must be specified"
  162. sys.exit(1)
  163. installed_filepath = None
  164. if download:
  165. if not options.buildnumber:
  166. if options.buildconfig == "release":
  167. options.buildnumber = default_release_build_number
  168. else:
  169. options.buildnumber = default_debug_build_number
  170. if not options.dir:
  171. if options.buildconfig == "release":
  172. options.dir = default_release_dir
  173. else:
  174. options.dir = default_debug_dir
  175. installed_filepath = os.path.join(options.output, "konvergo-depends-%s-%s-%s.installed" % (options.platform, options.buildconfig, options.buildnumber))
  176. if os.path.exists(installed_filepath) and options.buildnumber != "latest":
  177. print "The required deps bundle was already downloaded and installed."
  178. print "You can delete %s to force a reinstall." % installed_filepath
  179. sys.exit(0)
  180. # Delete previous installed stamps
  181. cont = ["-c"]
  182. for path in glob.iglob(os.path.join(options.output, "konvergo-depends-%s-%s-*.installed") % (options.platform, options.buildconfig)):
  183. match = re.search("-(\d+)\.installed", path, re.DOTALL)
  184. if match:
  185. if not match.group(1) == options.buildnumber:
  186. cont = []
  187. os.remove(path)
  188. if options.tag == "auto":
  189. req = urllib2.Request("https://nightlies.plex.tv/directdl/plex-dependencies/%s/%s/hash.txt" % (options.dir, options.buildnumber))
  190. try:
  191. match = urllib2.urlopen(req).read().rstrip()
  192. except urllib2.URLError, err:
  193. print err
  194. print "ERROR - Download failed"
  195. sys.exit(1)
  196. options.tag = match
  197. base_filename = "konvergo-depends-%s-%s-%s" % (options.platform, options.buildconfig, options.tag)
  198. filename = "%s.tbz2" % base_filename
  199. installed_filepath = os.path.join(options.output, "konvergo-depends-%s-%s-%s.installed" % (options.platform, options.buildconfig, options.buildnumber))
  200. if os.path.exists(installed_filepath):
  201. print "%s was already downloaded and installed." % filename
  202. print "You can delete %s to force a reinstall." % installed_filepath
  203. sys.exit(0)
  204. url = "https://nightlies.plex.tv/directdl/plex-dependencies/%s/%s/%s" % (options.dir, options.buildnumber, filename)
  205. inputfile = os.path.join(options.output, filename)
  206. print "-- Downloading %s ..." % url
  207. exec_cmd(["wget", "--no-check-certificate"] + cont + ["-O", inputfile, url])
  208. shaurl = "%s.sha.txt" % url
  209. shafile = "%s.sha.txt" % inputfile
  210. print "-- Downloading %s ..." % shaurl
  211. exec_cmd(["wget", "--no-check-certificate"] + ["-O", shafile, shaurl])
  212. else:
  213. inputfile = options.inputfile
  214. # Unpack and install
  215. packagename = unpack_and_install(download, inputfile, installed_filepath)
  216. # On OS X, we need to postprocess the dependencies.
  217. if platform.system() == 'Darwin':
  218. root = os.path.realpath(os.path.join(os.getcwd()))
  219. script = os.path.join(root, "scripts", "fix-install-names.py")
  220. for p in ("lib", "bin", "update_installer"):
  221. path = os.path.join(root, "Dependencies", "konvergo-depends-" + options.platform + "-" + options.buildconfig, p)
  222. exec_cmd([script, path])
  223. # Done!
  224. print "-- Done with %s" % packagename