annotate xena_import.py @ 45:129ba676fd68

remove more strange xmls
author jingchunzhu <jingchunzhu@gmail.com>
date Mon, 27 Jul 2015 15:39:35 -0700
parents 1ef1886dae04
children 8da6920a39ac
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
1 #!/usr/bin/env python
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
2
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
3 """
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
4 xena_import.py: import a dataset into Xena
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
5
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
6 Given a cmdline-specified genomic data file and a cmdline-specified Xena
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
7 directory, import the genomic data fle into Xena. This requires assembling
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
8 the necessary json file, based on cmdline input.
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
9 """
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
10
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
11 import argparse
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
12 import json
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
13 import os
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
14 import re
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
15 import shutil
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
16 import subprocess
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
17 import sys
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
18 import traceback
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
19 import xena_utils as xena
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
20
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
21 def updateColNormalization(jsonMetadata):
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
22 """Set colNormalization to TRUE if the data is of some expression-related
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
23 subtype and if colNormalization has not been set"""
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
24 if not jsonMetadata.has_key("colNormalization"):
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
25 if jsonMetadata.has_key("dataSubType"):
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
26 if re.search("expression", jsonMetadata['dataSubType'],
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
27 re.IGNORECASE):
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
28 jsonMetadata["colNormalization"] = "true"
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
29
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
30 def verifyAndImportExistingMetadata(inputJsonPath, outputJsonPath):
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
31 """Take an existing JSON file. Read the contents, and check for any
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
32 content that might be invalid in the local installation. Delete that
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
33 content. Write the rest to the indicated output pathname"""
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
34 problemFields = [":probeMap"]
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
35 fileContents = open(inputJsonPath).read()
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
36 jsonMetadata = json.loads(fileContents)
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
37 for thisProblem in problemFields:
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
38 if jsonMetadata.has_key(thisProblem):
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
39 del jsonMetadata[thisProblem]
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
40 updateColNormalization(jsonMetadata)
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
41 fp = open(outputJsonPath, "w")
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
42 fp.write("%s\n" % json.dumps(jsonMetadata, indent=2))
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
43 fp.close()
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
44
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
45
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
46 def main():
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
47 parser = argparse.ArgumentParser()
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
48 parser.add_argument("genomicDataPath", type=str)
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
49 parser.add_argument("outfile", type=str)
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
50 parser.add_argument("--json", type=str, default=None)
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
51 parser.add_argument("--cohort", type=str)
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
52 parser.add_argument("--type", type=str)
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
53 parser.add_argument("--dataSubType", type=str, default=None)
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
54 parser.add_argument("--label", type=str, default=None)
2
6eeb3ca663fb Fixed dataset delete bugs (the full pathname was needed) plus changed xena_import to give a checkbox for turning on column normalization
melissacline
parents: 0
diff changeset
55 parser.add_argument("--colNormalization", type=bool, default=False)
35
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
56 parser.add_argument("--probeMap", type=str, default=None)
0
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
57 args = parser.parse_args()
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
58
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
59 fp2 = open(args.outfile, "w")
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
60 fp2.write("Importing data to Xena\n")
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
61 xenaFileDir = xena.fileDir()
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
62 genomicDataFile = args.genomicDataPath.split("/")[-1]
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
63 jsonMetadataTargetPathname = "%s/%s.json" % (xenaFileDir,
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
64 genomicDataFile)
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
65
35
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
66 if args.probeMap is not None:
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
67 probeMapDataFile = args.probeMap.split("/")[-1]
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
68 jsonProbeMapMetadataTargetPathname = "%s/%s.json" % (xenaFileDir,
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
69 probeMapDataFile)
0
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
70
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
71 # The metadata either came as the name of a JSON file or a series of
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
72 # command line arguments.
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
73 if args.json is not None:
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
74 # In this case, the metadata came in the form of a JSON file.
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
75 # Verify that the metadata is valid on the current system, which
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
76 # might mean altering it. Import the stuff that will validate.
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
77 verifyAndImportExistingMetadata(args.json, jsonMetadataTargetPathname)
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
78 else:
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
79 # In this case, the metadata came in the form of a series of
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
80 # command line arguments. Assemble them into JSON format,
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
81 # and write a JSON file into the Xena file directory.
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
82 metadata = { 'cohort': args.cohort, 'type': args.type }
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
83 if args.dataSubType is not None:
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
84 metadata['dataSubType'] = args.dataSubType
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
85 if args.label is not None:
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
86 metadata['label'] = args.label
2
6eeb3ca663fb Fixed dataset delete bugs (the full pathname was needed) plus changed xena_import to give a checkbox for turning on column normalization
melissacline
parents: 0
diff changeset
87 if args.colNormalization:
6eeb3ca663fb Fixed dataset delete bugs (the full pathname was needed) plus changed xena_import to give a checkbox for turning on column normalization
melissacline
parents: 0
diff changeset
88 metadata['colNormalization'] = "true"
6eeb3ca663fb Fixed dataset delete bugs (the full pathname was needed) plus changed xena_import to give a checkbox for turning on column normalization
melissacline
parents: 0
diff changeset
89 else:
6eeb3ca663fb Fixed dataset delete bugs (the full pathname was needed) plus changed xena_import to give a checkbox for turning on column normalization
melissacline
parents: 0
diff changeset
90 updateColNormalization(metadata)
35
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
91 if args.probeMap is not None:
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
92 metadata[':probeMap'] = probeMapDataFile
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
93
0
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
94 jsonMetadata = json.dumps(metadata, indent=2)
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
95 fp = open(jsonMetadataTargetPathname, "w")
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
96 fp.write("%s\n" % (jsonMetadata))
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
97 fp.close()
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
98
35
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
99 if args.probeMap is not None:
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
100 probeMapMetadata={"type":"probeMap"}
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
101 jsonProbeMapMetadata = json.dumps(probeMapMetadata, indent=2)
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
102 jsonProbeMapMetadataTargetPathname = "%s/%s.json" % (xenaFileDir,
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
103 probeMapDataFile)
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
104 fp = open(jsonProbeMapMetadataTargetPathname, "w")
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
105 fp.write("%s\n" % ( jsonProbeMapMetadata))
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
106 fp.close()
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
107
0
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
108 # Finally, copy the genomic data into the Xena directory
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
109 shutil.copy(args.genomicDataPath, xenaFileDir)
35
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
110 if args.probeMap is not None:
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
111 shutil.copy(args.probeMap, xenaFileDir)
0
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
112
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
113 # Set up the xena load comamnd and try to execute it. If an exception
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
114 # is generated, output a traceback and exit with nonzero status. If
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
115 # no exception was generated, indicate a successful import and exit
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
116 # with zero status.
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
117 xenaLoadCmd = "java -jar %s -l --force %s/%s -p %s" % (xena.jarPath(),
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
118 xenaFileDir,
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
119 genomicDataFile,
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
120 xena.port())
38
1ef1886dae04 data import sucess/unsuccess message more informative
jingchunzhu
parents: 36
diff changeset
121 if args.label is not None:
1ef1886dae04 data import sucess/unsuccess message more informative
jingchunzhu
parents: 36
diff changeset
122 name = args.label
1ef1886dae04 data import sucess/unsuccess message more informative
jingchunzhu
parents: 36
diff changeset
123 else:
1ef1886dae04 data import sucess/unsuccess message more informative
jingchunzhu
parents: 36
diff changeset
124 name = genomicDataFile
1ef1886dae04 data import sucess/unsuccess message more informative
jingchunzhu
parents: 36
diff changeset
125 loadInToXena (xenaLoadCmd, name, fp2)
1ef1886dae04 data import sucess/unsuccess message more informative
jingchunzhu
parents: 36
diff changeset
126
35
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
127 if args.probeMap is not None:
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
128 xenaLoadCmd = "java -jar %s -l --force %s/%s -p %s" % (xena.jarPath(),
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
129 xenaFileDir,
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
130 probeMapDataFile,
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
131 xena.port())
38
1ef1886dae04 data import sucess/unsuccess message more informative
jingchunzhu
parents: 36
diff changeset
132 loadInToXena (xenaLoadCmd, "id to gene mapping file for %s" % (name), fp2)
36
jingchunzhu
parents: 35
diff changeset
133 fp2.close()
35
d8dc482ef970 add functionality to import probeMap
jingchunzhu <jingchunzhu@gmail.com>
parents: 2
diff changeset
134
36
jingchunzhu
parents: 35
diff changeset
135 def loadInToXena (xenaLoadCmd, name, fp2):
jingchunzhu
parents: 35
diff changeset
136 print xenaLoadCmd
0
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
137 try:
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
138 subprocess.call(xenaLoadCmd, shell=True)
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
139 except:
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
140 exc_type, exc_value, exc_traceback = sys.exc_info()
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
141 lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
142 allLines = ''.join('!! ' + line for line in lines)
36
jingchunzhu
parents: 35
diff changeset
143 fp2.write("Unsuccessful: Import %s\n" % (name))
0
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
144 fp2.write("Unsuccessful: error %s\n" % allLines)
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
145 else:
38
1ef1886dae04 data import sucess/unsuccess message more informative
jingchunzhu
parents: 36
diff changeset
146 fp2.write( "Importing %s is successful\n" % (name))
0
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
147
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
148 if __name__ == "__main__":
8bb037f88ed2 Uploaded
melissacline
parents:
diff changeset
149 main()