fixed mp3 format issue

app.py

@@ -1,119 +1,138 @@
+import io
 import whisper
 import torch
+import ffmpeg
 import torchaudio
 import streamlit as st
-
 LANGUAGES = {
-    "english":"en",
-    "chinese":"zh",
-    "german":"de",
-    "spanish":"es",
-    "russian":"ru",
-    "korean":"ko",
-    "french":"fr",
-    "japanese":"ja",
-    "portuguese":"pt",
-    "turkish":"tr",
-    "polish":"pl",
-    "catalan":"ca",
-    "dutch":"nl",
-    "arabic":"ar",
-    "swedish":"sv",
-    "italian":"it",
-    "indonesian":"id",
-    "hindi":"hi",
-    "finnish":"fi",
-    "vietnamese":"vi",
-    "hebrew":"iw",
-    "ukrainian":"uk",
-    "greek":"el",
-    "malay":"ms",
-    "czech":"cs",
-    "romanian":"ro",
-    "danish":"da",
-    "hungarian":"hu",
-    "tamil":"ta",
-    "norwegian":"no",
-    "thai":"th",
-    "urdu":"ur",
-    "croatian":"hr",
-    "bulgarian":"bg",
-    "lithuanian":"lt",
-    "latin":"la",
-    "maori":"mi",
-    "malayalam":"ml",
-    "welsh":"cy",
-    "slovak":"sk",
-    "telugu":"te",
-    "persian":"fa",
-    "latvian":"lv",
-    "bengali":"bn",
-    "serbian":"sr",
-    "azerbaijani":"az",
-    "slovenian":"sl",
-    "kannada":"kn",
-    "estonian":"et",
-    "macedonian":"mk",
-    "breton":"br",
-    "basque":"eu",
-    "icelandic":"is",
-    "armenian":"hy",
-    "nepali":"ne",
-    "mongolian":"mn",
-    "bosnian":"bs",
-    "kazakh":"kk",
-    "albanian":"sq",
-    "swahili":"sw",
-    "galician":"gl",
-    "marathi":"mr",
-    "punjabi":"pa",
-    "sinhala":"si",
-    "khmer":"km",
-    "shona":"sn",
-    "yoruba":"yo",
-    "somali":"so",
-    "afrikaans":"af",
-    "occitan":"oc",
-    "georgian":"ka",
-    "belarusian":"be",
-    "tajik":"tg",
-    "sindhi":"sd",
-    "gujarati":"gu",
-    "amharic":"am",
-    "yiddish":"yi",
-    "lao":"lo",
-    "uzbek":"uz",
-    "faroese":"fo",
-    "haitian creole":"ht",
-    "pashto":"ps",
-    "turkmen":"tk",
-    "nynorsk":"nn",
-    "maltese":"mt",
-    "sanskrit":"sa",
-    "luxembourgish":"lb",
-    "myanmar":"my",
-    "tibetan":"bo",
-    "tagalog":"tl",
-    "malagasy":"mg",
-    "assamese":"as",
-    "tatar":"tt",
-    "hawaiian":"haw",
-    "lingala":"ln",
-    "hausa":"ha",
-    "bashkir":"ba",
-    "javanese":"jw",
-    "sundanese":"su",
+    "en":"english",
+    "zh":"chinese",
+    "de":"german",
+    "es":"spanish",
+    "ru":"russian",
+    "ko":"korean",
+    "fr":"french",
+    "ja":"japanese",
+    "pt":"portuguese",
+    "tr":"turkish",
+    "pl":"polish",
+    "ca":"catalan",
+    "nl":"dutch",
+    "ar":"arabic",
+    "sv":"swedish",
+    "it":"italian",
+    "id":"indonesian",
+    "hi":"hindi",
+    "fi":"finnish",
+    "vi":"vietnamese",
+    "iw":"hebrew",
+    "uk":"ukrainian",
+    "el":"greek",
+    "ms":"malay",
+    "cs":"czech",
+    "ro":"romanian",
+    "da":"danish",
+    "hu":"hungarian",
+    "ta":"tamil",
+    "no":"norwegian",
+    "th":"thai",
+    "ur":"urdu",
+    "hr":"croatian",
+    "bg":"bulgarian",
+    "lt":"lithuanian",
+    "la":"latin",
+    "mi":"maori",
+    "ml":"malayalam",
+    "cy":"welsh",
+    "sk":"slovak",
+    "te":"telugu",
+    "fa":"persian",
+    "lv":"latvian",
+    "bn":"bengali",
+    "sr":"serbian",
+    "az":"azerbaijani",
+    "sl":"slovenian",
+    "kn":"kannada",
+    "et":"estonian",
+    "mk":"macedonian",
+    "br":"breton",
+    "eu":"basque",
+    "is":"icelandic",
+    "hy":"armenian",
+    "ne":"nepali",
+    "mn":"mongolian",
+    "bs":"bosnian",
+    "kk":"kazakh",
+    "sq":"albanian",
+    "sw":"swahili",
+    "gl":"galician",
+    "mr":"marathi",
+    "pa":"punjabi",
+    "si":"sinhala",
+    "km":"khmer",
+    "sn":"shona",
+    "yo":"yoruba",
+    "so":"somali",
+    "af":"afrikaans",
+    "oc":"occitan",
+    "ka":"georgian",
+    "be":"belarusian",
+    "tg":"tajik",
+    "sd":"sindhi",
+    "gu":"gujarati",
+    "am":"amharic",
+    "yi":"yiddish",
+    "lo":"lao",
+    "uz":"uzbek",
+    "fo":"faroese",
+    "ht":"haitian creole",
+    "ps":"pashto",
+    "tk":"turkmen",
+    "nn":"nynorsk",
+    "mt":"maltese",
+    "sa":"sanskrit",
+    "lb":"luxembourgish",
+    "my":"myanmar",
+    "bo":"tibetan",
+    "tl":"tagalog",
+    "mg":"malagasy",
+    "as":"assamese",
+    "tt":"tatar",
+    "haw":"hawaiian",
+    "ln":"lingala",
+    "ha":"hausa",
+    "ba":"bashkir",
+    "jw":"javanese",
+    "su":"sundanese",
 }
 
 def decode(model, mel, options):
     result = whisper.decode(model, mel, options)
     return result.text
 
-def load_audio(path):
-    waveform, sample_rate = torchaudio.load(path)
-    if sample_rate != 16000:
-        waveform = torchaudio.transforms.Resample(sample_rate, 16000)(waveform)
-    return waveform.squeeze(0)
+def load_audio(audio):
+    print(audio.type)
+    if audio.type == "audio/wav" or audio.type == "audio/flac":
+        wave, sr = torchaudio.load(audio)
+        if sr != 16000:
+            wave = torchaudio.transforms.Resample(sr, 16000)(wave)
+        return wave.squeeze(0)
+
+    elif audio.type == "audio/mpeg":
+        audio = audio.read()
+        audio, _ = (ffmpeg
+            .input('pipe:0')
+            .output('pipe:1', format='wav', acodec='pcm_s16le', ac=1, ar='16k')
+            .run(capture_stdout=True, input=audio)
+        )
+        audio = io.BytesIO(audio)
+        wave, sr = torchaudio.load(audio)
+        if sr != 16000:
+            wave = torchaudio.transforms.Resample(sr, 16000)(wave)
+        return wave.squeeze(0)
+
+    else:
+        st.error("Unsupported audio format")
 
 def detect_language(model, mel):
     _, probs = model.detect_language(mel)
@@ -136,7 +155,7 @@ def main():
     st.sidebar.write(f"Model: {model_selection+' (Multilingual)' if not en_model_selection else model_selection + ' (English only)'}")
 
     if st.sidebar.checkbox("Show supported languages", value=False):
-            st.sidebar.info(list(LANGUAGES.keys()))
+            st.sidebar.info(list(LANGUAGES.values()))
     st.sidebar.title("Options")
     
     beam_size = st.sidebar.slider("Beam Size", min_value=1, max_value=10, value=5)
@@ -151,7 +170,7 @@ def main():
     audio_file = st.file_uploader("Upload Audio", type=["wav", "mp3", "flac"])
 
     if audio_file is not None:
-        st.audio(audio_file, format='audio/ogg')
+        st.audio(audio_file, format=audio_file.type)
         with st.spinner("Loading model..."):
             model = whisper.load_model(model_selection)
             model = model.to("cpu") if not torch.cuda.is_available() else model.to("cuda")
@@ -164,7 +183,7 @@ def main():
         if not en_model_selection:
             with st.spinner("Detecting language..."):
                 language = detect_language(model, mel)
-                st.markdown(f"Detected Language: {language}")
+                st.markdown(f"Detected Language: {LANGUAGES[language]} ({language})")
         else:
             language = "en"
         configuration = {"beam_size": beam_size, "fp16": fp16, "task": task, "language": language}

requirements.txt

@@ -3,6 +3,7 @@ numpy
 torch
 torchaudio
 tqdm
+ffmpeg-python
 more-itertools
 transformers>=4.19.0
 ffmpeg-python==0.2.0