Mercurial > repos > bgruening > sklearn_numeric_clustering

--- a/main_macros.xml	Tue Jul 10 03:06:37 2018 -0400
+++ b/main_macros.xml	Fri Jul 13 03:50:30 2018 -0400
@@ -35,7 +35,8 @@
     if not options['threshold'] or options['threshold'] == 'None':
       options['threshold'] = None
       if 'extra_estimator' in inputs and inputs['extra_estimator']['has_estimator'] == 'no_load':
-        fitted_estimator = pickle.load(open("inputs['extra_estimator']['fitted_estimator']", 'r'))
+        with open("inputs['extra_estimator']['fitted_estimator']", 'rb') as model_handler:
+          fitted_estimator = pickle.load(model_handler)
         new_selector = selector(fitted_estimator, prefit=True, **options)
       else:
         estimator=inputs["estimator"]
@@ -83,7 +84,7 @@
       parse_dates=True
     )
   else:
-    X = mmread(open(file1, 'r'))
+    X = mmread(file1)

   header = 'infer' if params["selected_tasks"]["selected_algorithms"]["input_options"]["header2"] else None
   column_option = params["selected_tasks"]["selected_algorithms"]["input_options"]["column_selector_options_2"]["selected_column_selector_option2"]
@@ -432,19 +433,6 @@


   <!--Data interface-->
-  <xml name="tabular_input">
-    <param name="infile" type="data" format="tabular" label="Data file with numeric values"/>
-    <param name="start_column" type="data_column" data_ref="infile" optional="True" label="Select a subset of data. Start column:" />
-    <param name="end_column" type="data_column" data_ref="infile" optional="True" label="End column:" />
-  </xml>
-
-  <xml name="sample_cols" token_label1="File containing true class labels:" token_label2="File containing predicted class labels:" token_multiple1="False" token_multiple2="False" token_format1="tabular" token_format2="tabular" token_help1="" token_help2="">
-    <param name="infile1" type="data" format="@FORMAT1@" label="@LABEL1@" help="@HELP1@"/>
-    <param name="col1" multiple="@MULTIPLE1@" type="data_column" data_ref="infile1" label="Select target column(s):"/>
-    <param name="infile2" type="data" format="@FORMAT2@" label="@LABEL2@" help="@HELP2@"/>
-    <param name="col2" multiple="@MULTIPLE2@" type="data_column" data_ref="infile2" label="Select target column(s):"/>
-    <yield/>
-  </xml>

   <xml name="samples_tabular" token_multiple1="false" token_multiple2="false">
     <param name="infile1" type="data" format="tabular" label="Training samples dataset:"/>
@@ -472,13 +460,13 @@
       <param name="@COL_NAME@" multiple="@MULTIPLE@" type="data_column" data_ref="@INFILE@" label="Select target column(s):"/>
     </when>
     <when value="by_header_name">
-      <param name="@COL_NAME@" type="text" value="" label="Type header name(s):" help="String seperate by colon. For example: target1,target2"/>
+      <param name="@COL_NAME@" type="text" value="" label="Type header name(s):" help="Comma-separated string. For example: target1,target2"/>
     </when>
     <when value="all_but_by_index_number">
       <param name="@COL_NAME@" multiple="@MULTIPLE@" type="data_column" data_ref="@INFILE@" label="Select target column(s):"/>
     </when>
     <when value="all_but_by_header_name">
-      <param name="@COL_NAME@" type="text" value="" label="Type header name(s):" help="String seperate by colon. For example: target1,target2"/>
+      <param name="@COL_NAME@" type="text" value="" label="Type header name(s):" help="Comma-separated string. For example: target1,target2"/>
     </when>
     <when value="all_columns">
     </when>
@@ -553,11 +541,6 @@
     </conditional>
   </xml>

-  <xml name="multitype_input" token_format="tabular" token_help="All datasets with tabular format are supporetd.">
-    <param name="infile_transform" type="data" format="@FORMAT@" label="Select a dataset to transform:" help="@HELP@"/>
-  </xml>
-
-
   <!--Advanced options-->
   <xml name="nn_advanced_options">
     <section name="options" title="Advanced Options" expanded="False">
@@ -822,9 +805,17 @@
     </param>
   </xml>

+  <xml name="sparse_preprocessors_ext">
+    <expand macro="sparse_preprocessors">
+      <option value="KernelCenterer">Kernel Centerer (Centers a kernel matrix)</option>
+      <option value="MinMaxScaler">Minmax Scaler (Scales features to a range)</option>
+      <option value="PolynomialFeatures">Polynomial Features (Generates polynomial and interaction features)</option>
+      <option value="RobustScaler">Robust Scaler (Scales features using outlier-invariance statistics)</option>
+    </expand>
+  </xml>
+
   <xml name="sparse_preprocessor_options">
     <when value="Binarizer">
-        <expand macro="multitype_input" format="tabular,txt" help="Tabular and sparse datasets are supporetd."/>
         <section name="options" title="Advanced Options" expanded="False">
             <param argument="copy" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolfalse" checked="true"
                 label="Use a copy of data for precomputing binarization" help=" "/>
@@ -834,7 +825,6 @@
         </section>
     </when>
     <when value="Imputer">
-      <expand macro="multitype_input" format="tabular,txt" help="Tabular and sparse datasets are supporetd."/>
       <section name="options" title="Advanced Options" expanded="False">
           <param argument="copy" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolfalse" checked="true"
             label="Use a copy of data for precomputing imputation" help=" "/>
@@ -854,7 +844,6 @@
       </section>
     </when>
     <when value="StandardScaler">
-      <expand macro="multitype_input"/>
       <section name="options" title="Advanced Options" expanded="False">
         <param argument="copy" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolfalse" checked="true"
             label="Use a copy of data for performing inplace scaling" help=" "/>
@@ -865,14 +854,12 @@
       </section>
     </when>
     <when value="MaxAbsScaler">
-      <expand macro="multitype_input" format="tabular,txt" help="Tabular and sparse datasets are supporetd."/>
       <section name="options" title="Advanced Options" expanded="False">
         <param argument="copy" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolfalse" checked="true"
             label="Use a copy of data for precomputing scaling" help=" "/>
       </section>
     </when>
     <when value="Normalizer">
-      <expand macro="multitype_input" format="tabular,txt" help="Tabular and sparse datasets are supporetd."/>
       <section name="options" title="Advanced Options" expanded="False">
         <param argument="norm" type="select" optional="true" label="The norm to use to normalize non zero samples" help=" ">
           <option value="l1" selected="true">l1</option>
@@ -885,6 +872,41 @@
     </when>
     <yield/>
   </xml>
+
+  <xml name="sparse_preprocessor_options_ext">
+    <expand macro="sparse_preprocessor_options">
+      <when value="KernelCenterer">
+        <section name="options" title="Advanced Options" expanded="False">
+        </section>
+      </when>
+      <when value="MinMaxScaler">
+          <section name="options" title="Advanced Options" expanded="False">
+              <!--feature_range-->
+              <param argument="copy" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolflase" checked="true"
+                  label="Use a copy of data for precomputing normalization" help=" "/>
+          </section>
+      </when>
+      <when value="PolynomialFeatures">
+          <section name="options" title="Advanced Options" expanded="False">
+              <param argument="degree" type="integer" optional="true" value="2" label="The degree of the polynomial features " help=""/>
+              <param argument="interaction_only" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolflase" checked="false" label="Produce interaction features only" help="(Features that are products of at most degree distinct input features) "/>
+              <param argument="include_bias" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolflase" checked="true" label="Include a bias column" help="Feature in which all polynomial powers are zero "/>
+          </section>
+      </when>
+      <when value="RobustScaler">
+          <section name="options" title="Advanced Options" expanded="False">
+              <!--=True, =True, copy=True-->
+              <param argument="with_centering" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolflase" checked="true"
+                  label="Center the data before scaling" help=" "/>
+              <param argument="with_scaling" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolflase" checked="true"
+                  label="Scale the data to interquartile range" help=" "/>
+              <param argument="copy" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolflase" checked="true"
+                  label="Use a copy of data for inplace scaling" help=" "/>
+          </section>
+      </when>
+    </expand>
+  </xml>
+
   <xml name="estimator_input_no_fit">
     <expand macro="feature_selection_estimator" />
     <conditional name="extra_estimator">
@@ -892,6 +914,7 @@
       <expand macro="feature_selection_estimator_choices" />
     </conditional>
   </xml>
+
   <xml name="feature_selection_all">
     <conditional name="feature_selection_algorithms">
       <param name="selected_algorithm" type="select" label="Select a feature selection algorithm">
@@ -1014,6 +1037,7 @@
       </when-->
     </conditional>
   </xml>
+
   <xml name="feature_selection_score_function">
     <param argument="score_func" type="select" label="Select a score function">
       <option value="chi2">chi2 - Compute chi-squared stats between each non-negative feature and class</option>
@@ -1023,6 +1047,7 @@
       <option value="mutual_info_regression">mutual_info_regression - Estimate mutual information for a continuous target variable</option>
     </param>
   </xml>
+
   <xml name="feature_selection_estimator">
     <param argument="estimator" type="select" label="Select an estimator" help="The base estimator from which the transformer is built.">
       <option value="svm.SVR(kernel=&quot;linear&quot;)">svm.SVR(kernel=&quot;linear&quot;)</option>
@@ -1032,6 +1057,7 @@
       <option value="ensemble.RandomForestRegressor(n_estimators = 1000, random_state = 42)">ensemble.RandomForestRegressor(n_estimators = 1000, random_state = 42)</option>
     </param>
   </xml>
+
   <xml name="feature_selection_extra_estimator">
       <param name="has_estimator" type="select" label="Does your estimator on the list above?">
         <option value="yes">Yes, my estimator is on the list</option>
@@ -1039,6 +1065,7 @@
         <yield/>
       </param>
   </xml>
+
   <xml name="feature_selection_estimator_choices">
     <when value="yes">
     </when>
@@ -1047,6 +1074,7 @@
     </when>
     <yield/>
   </xml>
+
   <xml name="feature_selection_methods">
     <conditional name="select_methods">
       <param name="selected_method" type="select" label="Select an operation">
--- a/numeric_clustering.xml	Tue Jul 10 03:06:37 2018 -0400
+++ b/numeric_clustering.xml	Fri Jul 13 03:50:30 2018 -0400
@@ -25,7 +25,9 @@
 @COLUMNS_FUNCTION@

 input_json_path = sys.argv[1]
-params = json.load(open(input_json_path, "r"))
+with open(input_json_path, "r") as param_handler:
+    params = json.load(param_handler)
+

 selected_algorithm = params["input_types"]["algorithm_options"]["selected_algorithm"]

@@ -36,7 +38,7 @@
 cluster_object.set_params(**options)

 #if $input_types.selected_input_type == "sparse":
-data_matrix = mmread(open("$infile", 'r'))
+data_matrix = mmread("$infile")
 #else:
 data = pandas.read_csv("$infile", sep='\t', header=0, index_col=None, parse_dates=True, encoding=None, tupleize_cols=False )
 header = 'infer' if params["input_types"]["header"] else None
--- a/test-data/mv_result07.tabular	Tue Jul 10 03:06:37 2018 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,1 +0,0 @@
-0.7824428015300172