donné une erreur unicode Je ne comprends pas

Question

Voici mon code, je suis sûr que ça a l'air terrible, mais tout cela fonctionne comme il se doit, seul problème que je vais avoir est la dernière ligne ...

import pyPdf 
import os 
import csv 

class UnicodeWriter: 
    """ 
    A CSV writer which will write rows to CSV file "f", 
    which is encoded in the given encoding. 
    """ 

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): 
     # Redirect output to a queue 
     self.queue = cStringIO.StringIO() 
     self.writer = csv.writer(self.queue, dialect=dialect, **kwds) 
     self.stream = f 
     self.encoder = codecs.getincrementalencoder(encoding)() 

    def writerow(self, row): 
     self.writer.writerow([s.encode("utf-8") for s in row]) 
     # Fetch UTF-8 output from the queue ... 
     data = self.queue.getvalue() 
     data = data.decode("utf-8") 
     # ... and reencode it into the target encoding 
     data = self.encoder.encode(data) 
     # write to the target stream 
     self.stream.write(data) 
     # empty queue 
     self.queue.truncate(0) 

    def writerows(self, rows): 
     for row in rows: 
      self.writerow(row) 


    PDFWriter = csv.writer(open('/home/nick/TAM_work/text/text.doc', 'a'), delimiter=' ', quotechar='|', quoting=csv.QUOTE_ALL) 

    def getPDFContent(path): 
     content = "" 
     # Load PDF into pyPDF 
     pdf = pyPdf.PdfFileReader(file(path, "rb")) 
     # Iterate pages 
     for i in range(0, pdf.getNumPages()): 
      # Extract text from page and add to content 
      content += pdf.getPage(i).extractText() + "\n" 
     # Collapse whitespace 
     content = " ".join(content.replace(u"\xa0", " ").strip().split()) 
     return content 

    for word in os.listdir("/home/nick/TAM_work/TAM_pdfs"): 
    print getPDFContent("/home/nick/TAM_work/TAM_pdfs/" + word) 

    PDFWriter.writerow ([getPDFContent("/home/nick/TAM_work/TAM_pdfs/" + word)]) 

Lorsque je l'exécute tout fonctionne jusqu'à ce qu'il frappe cette ...

Traceback (most recent call last): 
    File "Saving_fuction_added.py", line 52, in <module> 
    PDFWriter.writerow ([getPDFContent("/home/nick/TAM_work/TAM_pdfs/" + word)]) 
UnicodeEncodeError: 'ascii' codec can't encode character u'\u2122' in position 81: ordinal not in range(128) 

J'aimerais toute aide. Merci les gars.

Matt

Answer 1

Voici le code qui a répondu à cette question. Mais maintenant, il écrit seulement le dernier fichier.

import pyPdf 
import os 
import csv 

class UnicodeWriter: 
    """ 
    A CSV writer which will write rows to CSV file "f", 
    which is encoded in the given encoding. 
    """ 

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): 
     # Redirect output to a queue 
     self.queue = cStringIO.StringIO() 
     self.writer = csv.writer(self.queue, dialect=dialect, **kwds) 
     self.stream = f 
     self.encoder = codecs.getincrementalencoder(encoding)() 

    def writerow(self, row): 
     self.writer.writerow([s.encode("utf-8") for s in row]) 
     # Fetch UTF-8 output from the queue ... 
     data = self.queue.getvalue() 
     data = data.decode("utf-8") 
     # ... and reencode it into the target encoding 
     data = self.encoder.encode(data) 
     # write to the target stream 
     self.stream.write(data) 
     # empty queue 
     self.queue.truncate(0) 

    def writerows(self, rows): 
     for row in rows: 
      self.writerow(row) 


PDFWriter = csv.writer(open('/home/nick/TAM_work/text/text.doc', 'a'), delimiter=' ', quotechar='|', quoting=csv.QUOTE_ALL) 

def getPDFContent(path): 
    content = "" 
    # Load PDF into pyPDF 
    pdf = pyPdf.PdfFileReader(file(path, "rb")) 
    # Iterate pages 
    for i in range(0, pdf.getNumPages()): 
     # Extract text from page and add to content 
     content += pdf.getPage(i).extractText() + "\n" 
    # Collapse whitespace 
    content = " ".join(content.replace(u"\xa0", " ").strip().split()) 
    return content 

for word in os.listdir("/home/nick/TAM_work/TAM_pdfs"): 
    print getPDFContent("/home/nick/TAM_work/TAM_pdfs/" + word) 

PDFWriter.writerow ([getPDFContent("/home/nick/TAM_work/TAM_pdfs/" + word).encode("ascii", "ignore")]) 

Answer 2

comme sous-ordre Vous mettez un grand nombre dans une petite variable et son jet une exception.

Je vous présente un outil C# qui fonctionne très bien avec unicode, vous pouvez le trouver à http://unicode.codeplex.com

dans votre cas, je vous recommande de changer les

for i in range(0, pdf.getNumPages()): 

pdf.getNumPages() est au-dessus de 128 juste le contrôler.