Cloud Service Integration
Machine Learning
Streaming Data
Operationalization
Natural Language Processing
Bioinformatics and Genomics
Text Data Preparation
Tokenization and Vectorization
Load in required libraries
1
from pyspark.ml.feature import RegexTokenizer, StopWordsRemover, CountVectorizer
Copied!
Remove usernames, dates, links, etc.
1
2
def clean_text(c):
3
c = lower(c)
4
c = regexp_replace(c, "(https?\://)\S+", "") # Remove links
5
c = regexp_replace(c, "(\\n)|\n|\r|\t", "") # Remove CR, tab, and LR
6
c = regexp_replace(c, "(?:(?:[0-9]{2}[:\/,]){2}[0-9]{2,4})", "") # Remove dates
7
c = regexp_replace(c, "@([A-Za-z0-9_]+)", "") # Remove usernames
8
c = regexp_replace(c, "[0-9]", "") # Remove numbers
9
c = regexp_replace(c, "\:|\/|\#|\.|\?|\!|\&|\"|\,", "") # Remove symbols
10
#c = regexp_replace(c, "(@[A-Za-z0-9_]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)", "")
11
return c
12
13
dataset = dataset.withColumn("text", clean_text(col("text")))
Copied!
RegEx tokenization
1
regexTokenizer = RegexTokenizer(inputCol="text", outputCol="words", pattern="\\W")
Copied!
Remove stop words
1
# Add Stop words
2
add_stopwords = ["http","https","amp","rt","t","c","the","@","/",":"] # standard web stop words
3
4
stopwordsRemover = StopWordsRemover(inputCol="words", outputCol="filtered").setStopWords(add_stopwords)
Copied!
Count words
1
# Bag of Words Count
2
countVectors = CountVectorizer(inputCol="filtered", outputCol="features", vocabSize=10000, minDF=5)
Copied!
Index strings
1
# String Indexer
2
from pyspark.ml.feature import OneHotEncoder, StringIndexer, VectorAssembler
3
label_stringIdx = StringIndexer(inputCol = "class", outputCol = "label")
Copied!
Create transformation pipeline
1
from pyspark.ml import Pipeline
2
3
pipeline = Pipeline(stages=[regexTokenizer, stopwordsRemover, countVectors, label_stringIdx])
4
5
# Fit the pipeline to training documents.
6
pipelineFit = pipeline.fit(data)
7
dataset = pipelineFit.transform(data)
Copied!
Once the transformation pipeline has been fit, you can use normal classification algorithms for classifying the text.
Extras
Get label numbers for each class
1
from pyspark.sql import *
2
from pyspark.sql.functions import col
3
labelset = dataset.select(col("class"),
4
col("label")).distinct()
5
display(labelset)
Copied!
Split text body into sentences
1
from pyspark.sql.types import *
2
from pyspark.sql.window import *
3
from pyspark.sql.functions import col, split, explode, row_number
4
# Split text by sentence and convert to array
5
array_df = data.withColumn("text", split(col("text"), "\.").cast("array<string>"))
6
7
# Explode array into separate rows in the dataset
8
split_df = array_df.withColumn("text", explode(col("text")))\
9
.withColumn("part_number", row_number().over(Window.partitionBy("internet_message_id").orderBy("id")))
10
data = split_df
11
display(data)
Copied!
Create `part_number` for the split sentences
1
from pyspark.sql.window import *
2
from pyspark.sql.functions import row_number
3
4
data.withColumn("part_number", row_number().over(Window.partitionBy("body_id").orderBy("id"))).show()
Copied!