Xtykutl

import findspark

findspark.init()

import pyspark # only run after findspark.init()

from pyspark.sql import SparkSession

spark = SparkSession.builder.getOrCreate()

import pandas as pd

sc = spark.sparkContext



from pyspark.sql.window import Window

from pyspark.sql.functions import count, col

from pyspark.sql import Row

from pyspark.sql.functions import udf

def nonasciitoascii(unicodestring):

   return unicodestring.encode("ascii","ignore")



df=spark.read.csv("abc.csv")

df.show()



df.printSchema()



convertedudf = udf(nonasciitoascii)

converted = df.select('_c1','_c2').withColumn('converted',convertedudf(df._c1))

    converted.show()

import findspark

findspark.init()

import pyspark # only run after findspark.init()

from pyspark.sql import SparkSession

spark = SparkSession.builder.getOrCreate()

import pandas as pd

sc = spark.sparkContext



from pyspark.sql.window import Window

from pyspark.sql.functions import count, col

from pyspark.sql import Row

from pyspark.sql.functions import udf

def nonasciitoascii(unicodestring):

   return unicodestring.encode("ascii","ignore")



df=spark.read.csv("abc.csv")

df.show()



df.printSchema()



convertedudf = udf(nonasciitoascii)

converted = df.select('_c1','_c2').withColumn('converted',convertedudf(df._c1))

    converted.show()

import findspark

findspark.init()

import pyspark # only run after findspark.init()

from pyspark.sql import SparkSession

spark = SparkSession.builder.getOrCreate()

import pandas as pd

sc = spark.sparkContext



from pyspark.sql.window import Window

from pyspark.sql.functions import count, col

from pyspark.sql import Row

from pyspark.sql.functions import udf

def nonasciitoascii(unicodestring):

   return unicodestring.encode("ascii","ignore")



df=spark.read.csv("abc.csv")

df.show()



df.printSchema()



convertedudf = udf(nonasciitoascii)

converted = df.select('_c1','_c2').withColumn('converted',convertedudf(df._c1))

    converted.show()

import findspark

findspark.init()

import pyspark # only run after findspark.init()

from pyspark.sql import SparkSession

spark = SparkSession.builder.getOrCreate()

import pandas as pd

sc = spark.sparkContext



from pyspark.sql.window import Window

from pyspark.sql.functions import count, col

from pyspark.sql import Row

from pyspark.sql.functions import udf

def nonasciitoascii(unicodestring):

   return unicodestring.encode("ascii","ignore")



df=spark.read.csv("abc.csv")

df.show()



df.printSchema()



convertedudf = udf(nonasciitoascii)

converted = df.select('_c1','_c2').withColumn('converted',convertedudf(df._c1))

    converted.show()

add a comment | 

Your Answer

StackExchange.ifUsing("editor", function () {
StackExchange.using("externalEditor", function () {
StackExchange.using("snippets", function () {
StackExchange.snippets.init();
});
});
}, "code-snippets");

StackExchange.ready(function() {
var channelOptions = {
tags: "".split(" "),
id: "1"
};
initTagRenderer("".split(" "), "".split(" "), channelOptions);

StackExchange.using("externalEditor", function() {
// Have to fire editor after snippets, if snippets enabled
if (StackExchange.settings.snippets.snippetsEnabled) {
StackExchange.using("snippets", function() {
createEditor();
});
}
else {
createEditor();
}
});

function createEditor() {
StackExchange.prepareEditor({
heartbeatType: 'answer',
autoActivateHeartbeat: false,
convertImagesToLinks: true,
noModals: true,
showLowRepImageUploadWarning: true,
reputationToPostImages: 10,
bindNavPrevention: true,
postfix: "",
imageUploader: {
brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
allowUrls: true
},
onDemand: true,
discardSelector: ".discard-answer"
,immediatelyShowMarkdownHelp:true
});


}
});

active

oldest

votes

# in python 3

def check_ascii(string):

    if(not string.isascii()):

        return string

    else:

        return None



def check_ascii_in_python_2(string):

     if(not all(ord(char) < 128 for char in string)):

         return string

     else:

         return None



all_strings_with_non_ascii_chars = df.select('_c1','_c2').withColumn('check', check_ascii(df._c1)).filter('check is not null').select('check')

all_strings_with_non_ascii_chars.show()

# in python 3

def check_ascii(string):

    if(not string.isascii()):

        return string

    else:

        return None



def check_ascii_in_python_2(string):

     if(not all(ord(char) < 128 for char in string)):

         return string

     else:

         return None



all_strings_with_non_ascii_chars = df.select('_c1','_c2').withColumn('check', check_ascii(df._c1)).filter('check is not null').select('check')

all_strings_with_non_ascii_chars.show()

# in python 3

def check_ascii(string):

    if(not string.isascii()):

        return string

    else:

        return None



def check_ascii_in_python_2(string):

     if(not all(ord(char) < 128 for char in string)):

         return string

     else:

         return None



all_strings_with_non_ascii_chars = df.select('_c1','_c2').withColumn('check', check_ascii(df._c1)).filter('check is not null').select('check')

all_strings_with_non_ascii_chars.show()

# in python 3

def check_ascii(string):

    if(not string.isascii()):

        return string

    else:

        return None



def check_ascii_in_python_2(string):

     if(not all(ord(char) < 128 for char in string)):

         return string

     else:

         return None



all_strings_with_non_ascii_chars = df.select('_c1','_c2').withColumn('check', check_ascii(df._c1)).filter('check is not null').select('check')

all_strings_with_non_ascii_chars.show()

add a comment |