Mercurial > repos > stevecassidy > nltktools
annotate g_stemmer.py @ 3:0df72a8ab095 draft default tip
planemo upload for repository https://github.com/Alveo/alveo-galaxy-tools commit f2432aaedd36ae7662873623d8861d0982dffdd2
| author | stevecassidy |
|---|---|
| date | Mon, 20 Nov 2017 22:52:11 -0500 |
| parents | a47980ef2b96 |
| children |
| rev | line source |
|---|---|
|
0
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
1 import nltk |
|
2
a47980ef2b96
planemo upload for repository https://github.com/Alveo/alveo-galaxy-tools commit b5b26e9118f2ad8af109d606746b39a5588f0511-dirty
stevecassidy
parents:
1
diff
changeset
|
2 from nltk.stem import PorterStemmer, LancasterStemmer, snowball |
|
0
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
3 import argparse |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
4 |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
5 |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
6 def arguments(): |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
7 parser = argparse.ArgumentParser(description="Segments the text input into separate sentences") |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
8 parser.add_argument('--input', required=True, action="store", type=str, help="input text file") |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
9 parser.add_argument('--output', required=True, action="store", type=str, help="output file path") |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
10 parser.add_argument('--stemmer', required=False, action="store", type=str, help="output file path") |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
11 args = parser.parse_args() |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
12 return args |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
13 |
|
2
a47980ef2b96
planemo upload for repository https://github.com/Alveo/alveo-galaxy-tools commit b5b26e9118f2ad8af109d606746b39a5588f0511-dirty
stevecassidy
parents:
1
diff
changeset
|
14 |
|
0
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
15 def stem_file(in_file, out_file, stemmer_type): |
|
1
fb617586f4b2
planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents:
0
diff
changeset
|
16 with open(in_file, 'r') as fd: |
|
fb617586f4b2
planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents:
0
diff
changeset
|
17 unsegmented = fd.read() |
|
fb617586f4b2
planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents:
0
diff
changeset
|
18 |
|
fb617586f4b2
planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents:
0
diff
changeset
|
19 with open(out_file, 'w') as output: |
|
fb617586f4b2
planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents:
0
diff
changeset
|
20 sentences = nltk.sent_tokenize(unsegmented) |
|
fb617586f4b2
planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents:
0
diff
changeset
|
21 stemmer = get_stemmer(stemmer_type) |
|
fb617586f4b2
planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents:
0
diff
changeset
|
22 for sentence in sentences: |
|
fb617586f4b2
planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents:
0
diff
changeset
|
23 words = nltk.word_tokenize(sentence) |
|
fb617586f4b2
planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents:
0
diff
changeset
|
24 for word in words: |
|
fb617586f4b2
planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents:
0
diff
changeset
|
25 stemmed_word = stemmer.stem(word) |
|
fb617586f4b2
planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents:
0
diff
changeset
|
26 output.write(stemmed_word) |
|
fb617586f4b2
planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents:
0
diff
changeset
|
27 output.write('\n') |
|
0
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
28 |
|
2
a47980ef2b96
planemo upload for repository https://github.com/Alveo/alveo-galaxy-tools commit b5b26e9118f2ad8af109d606746b39a5588f0511-dirty
stevecassidy
parents:
1
diff
changeset
|
29 |
|
0
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
30 def get_stemmer(stemmer_type): |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
31 if stemmer_type == 'lancaster': |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
32 stemmer = LancasterStemmer() |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
33 elif stemmer_type == 'porter': |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
34 stemmer = PorterStemmer() |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
35 else: |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
36 stemmer = snowball.EnglishStemmer() |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
37 return stemmer |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
38 |
|
2
a47980ef2b96
planemo upload for repository https://github.com/Alveo/alveo-galaxy-tools commit b5b26e9118f2ad8af109d606746b39a5588f0511-dirty
stevecassidy
parents:
1
diff
changeset
|
39 |
|
0
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
40 if __name__ == '__main__': |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
41 args = arguments() |
|
e991d4e60c17
planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff
changeset
|
42 stem_file(args.input, args.output, args.stemmer) |