public class TokenizeME
extends org.apache.pig.EvalFunc<org.apache.pig.data.DataBag>
Example:
define TokenizeME datafu.pig.text.opennlp.TokenizeME('data/en-token.bin');
-- input:
-- ("I believe the Masons have infiltrated the Apache PMC.")
infoo = LOAD 'input' AS (text:chararray);
-- output:
-- ({(I),(believe),(the),(Masons),(have),(infiltrated),(the),(Apache),(PMC),(.)})
outfoo = FOREACH infoo GENERATE TokenizeME(text) as tokens;
Constructor and Description |
---|
TokenizeME(java.lang.String modelPath) |
Modifier and Type | Method and Description |
---|---|
org.apache.pig.data.DataBag |
exec(org.apache.pig.data.Tuple input) |
java.util.List<java.lang.String> |
getCacheFiles() |
org.apache.pig.impl.logicalLayer.schema.Schema |
outputSchema(org.apache.pig.impl.logicalLayer.schema.Schema input) |
allowCompileTimeCalculation, finish, getArgToFuncMapping, getInputSchema, getLogger, getPigLogger, getReporter, getReturnType, getSchemaName, getSchemaType, getShipFiles, isAsynchronous, progress, setInputSchema, setPigLogger, setReporter, setUDFContextSignature, warn
public java.util.List<java.lang.String> getCacheFiles()
getCacheFiles
in class org.apache.pig.EvalFunc<org.apache.pig.data.DataBag>
public org.apache.pig.data.DataBag exec(org.apache.pig.data.Tuple input) throws java.io.IOException
exec
in class org.apache.pig.EvalFunc<org.apache.pig.data.DataBag>
java.io.IOException
public org.apache.pig.impl.logicalLayer.schema.Schema outputSchema(org.apache.pig.impl.logicalLayer.schema.Schema input)
outputSchema
in class org.apache.pig.EvalFunc<org.apache.pig.data.DataBag>