From: Erik Andresen Date: Sat, 20 Mar 2021 17:44:30 +0000 (+0100) Subject: added asr with vosk X-Git-Url: https://defiant.homedns.org/gitweb/?a=commitdiff_plain;h=00c6403ee3673ae0b1e4b064f65c56f6ce5bc33a;p=ros_wild_thumper.git added asr with vosk --- diff --git a/msg/Sensor.msg b/msg/Sensor.msg index 6f58c7a..23e4056 100644 --- a/msg/Sensor.msg +++ b/msg/Sensor.msg @@ -4,4 +4,5 @@ float32 temp # degree celsius uint8 humidity # percent float32 pressure # kph uint16 co +float32 dew_point # degree celsius bool ventilate diff --git a/scripts/asr_pocketsphinx.rb b/scripts/asr_pocketsphinx.rb new file mode 100755 index 0000000..70cd538 --- /dev/null +++ b/scripts/asr_pocketsphinx.rb @@ -0,0 +1,107 @@ +#!/usr/bin/ruby + +require 'gst' +require 'pry' +require 'logger' +require 'ros' +require 'std_msgs/String' + +class Speak + def initialize(node) + @logger = Logger.new(STDOUT) + @publisher = node.advertise('asr_result', Std_msgs::String) + @pipeline = Gst.parse_launch('alsasrc device="plughw:1,0" ! audio/x-raw,format=S16LE,channels=1,rate=16000 ! cutter leaky=true name=cutter'\ + ' ! tee name=jsgf ! queue leaky=downstream ! valve name=valve_jsgf drop=true ! pocketsphinx name=asr_jsgf ! fakesink async=false jsgf.'\ + ' ! pocketsphinx name=asr_kws ! fakesink async=false'\ + ) + # Ignore everything below the configured volume + cutter = @pipeline.get_by_name('cutter') + cutter.set_property('threshold-dB', -20) + cutter.set_property('pre-length', 100000000) # pocketsphinx needs about 0.1s before start + cutter.set_property('run-length', 1300000000) + + asr_jsgf = @pipeline.get_by_name('asr_jsgf') + asr_jsgf.set_property('hmm', 'pocketsphinx/adapt/cmusphinx-en-us-5.2') + asr_jsgf.set_property('mllr', 'pocketsphinx/adapt/mllr_matrix') + asr_jsgf.set_property('jsgf', 'data/robot.jsgf') + + asr_kws = @pipeline.get_by_name('asr_kws') + asr_kws.set_property('hmm', 'pocketsphinx/adapt/cmusphinx-en-us-5.2') + asr_kws.set_property('mllr', 'pocketsphinx/adapt/mllr_matrix') + asr_kws.set_property('kws', 'data/keywords.kws') + + bus = @pipeline.bus() + bus.add_watch do |bus, message| + case message.type + when Gst::MessageType::EOS + loop.quit + when Gst::MessageType::ERROR + p message.parse_error + binding.pry # open console + loop.quit + when Gst::MessageType::ELEMENT + if message.src.name == "asr_kws" + if message.structure.get_value(:final).value + keyword_detect(message.structure.get_value(:hypothesis).value, message.structure.get_value(:confidence).value) + end + elsif message.src.name == "asr_jsgf" + if message.structure.get_value(:final).value + final_result(message.structure.get_value(:hypothesis).value, message.structure.get_value(:confidence).value) + end + elsif message.src.name == "cutter" + if message.structure.get_value(:above).value + @logger.debug "Start recording.." + else + @logger.debug "Stop recording" + end + end + end + true + end + + @pipeline.play + end + + # Enables/Disables the jsgf pipeline branch + def enable_jsgf(bEnable) + valve = @pipeline.get_by_name('valve_jsgf') + valve.set_property("drop", !bEnable) + end + + # Result of jsgf pipeline branch + def final_result(hyp, confidence) + @logger.info "final: " + hyp + " " + confidence.to_s + enable_jsgf(false) + + # Publish pocketsphinx result as ros message + msg = Std_msgs::String.new + msg.data = hyp + @publisher.publish(msg) + end + + def keyword_detect(hyp, confidence) + @logger.debug "Got keyword: " + hyp + enable_jsgf(true) + end + + def stop + @pipeline.stop + end +end + +if __FILE__ == $0 + node = ROS::Node.new('asr_pocketsphinx') + app = Speak.new(node) + loop = GLib::MainLoop.new(nil, false) + begin + Thread.new { + loop.run + } + node.spin + rescue Interrupt + ensure + app.stop + node.shutdown + loop.quit + end +end diff --git a/scripts/asr_vosk.rb b/scripts/asr_vosk.rb new file mode 100755 index 0000000..bc73949 --- /dev/null +++ b/scripts/asr_vosk.rb @@ -0,0 +1,115 @@ +#!/usr/bin/ruby +# export AUDIODEV=plughw:CARD=ArrayUAC10,0 +# rec -q -t alsa -c 1 -b 16 -r 16000 -t wav - silence -l 1 0.1 0.3% -1 2.0 0.3% | ./asr_vosk.rb - + +require 'logger' +require 'websocket-eventmachine-client' +require 'json' +require 'ros' +require 'std_msgs/String' + +KEYWORDS = ["wild thumper"] +CONFIG = { + "config": { + "phrase_list": ["angle", "backward", "by", "centimeter", "compass", "current", "decrease", "default", "degree", "down", "eight", "eighteen", "eighty", "eleven", "fifteen", "fifty", "five", "forty", "forward", "four", "fourteen", "get", "go", "hundred", "increase", "left", "light", "lights", "meter", "mic", "minus", "motion", "mute", "nine", "nineteen", "ninety", "off", "on", "one", "position", "pressure", "right", "secure", "set", "seven", "seventeen", "seventy", "silence", "six", "sixteen", "sixty", "speed", "stop", "temp", "temperature", "ten", "thirteen", "thirty", "three", "to", "turn", "twelve", "twenty", "two", "up", "velocity", "voltage", "volume", "wild thumper", "zero"], + "sample_rate": 16000.0 + } +} + +class Speak + def initialize(node) + @logger = Logger.new(STDOUT) + @jsgf_enabled = false + @publisher = node.advertise('asr_result', Std_msgs::String) + + EM.run do + Signal.trap("INT") { send_eof } + @ws = WebSocket::EventMachine::Client.connect(:uri => 'ws://192.168.36.4:2700') + + def send_eof + @ws.send '{"eof" : 1}' + end + + def run + while true do + data = ARGF.read(16000) + if data + @ws.send data, :type => :binary + else + send_eof + break + end + end + end + + @ws.onopen do + @logger.info "Running.." + @ws.send CONFIG.to_json + + Thread.new { + run + } + end + + @ws.onmessage do |msg, type| + d = JSON.parse(msg) + handle_result(d) + end + + @ws.onclose do |code, reason| + puts "Disconnected with status code: #{code}" + exit + end + end + end + + def handle_result(msg) + if msg.has_key? "result" + msg["result"].each do |result| + @logger.debug "word=" + result["word"] + end + + text = msg["text"] + @logger.debug "text=" + msg["text"] + if KEYWORDS.include? text + keyword_detect(text) + return + end + if @jsgf_enabled + final_result(msg["text"]) + end + end + end + + # Enables/Disables the jsgf pipeline branch + def enable_jsgf(bEnable) + @jsgf_enabled = bEnable + end + + # Result of jsgf pipeline branch + def final_result(hyp) + @logger.info "final: " + hyp + enable_jsgf(false) + + # Publish pocketsphinx result as ros message + msg = Std_msgs::String.new + msg.data = hyp + @publisher.publish(msg) + end + + def keyword_detect(hyp) + @logger.debug "Got keyword: " + hyp + enable_jsgf(true) + end +end + +if __FILE__ == $0 + node = ROS::Node.new('asr_vosk') + app = Speak.new(node) + begin + node.spin + rescue Interrupt + ensure + node.shutdown + end +end diff --git a/scripts/pocketsphinx.rb b/scripts/pocketsphinx.rb deleted file mode 100755 index d8e7bef..0000000 --- a/scripts/pocketsphinx.rb +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/ruby - -require 'gst' -require 'pry' -require 'logger' -require 'ros' -require 'std_msgs/String' - -class Speak - def initialize(node) - @logger = Logger.new(STDOUT) - @publisher = node.advertise('asr_result', Std_msgs::String) - @pipeline = Gst.parse_launch('alsasrc device="plughw:1,0" ! audio/x-raw,format=S16LE,channels=1,rate=16000 ! cutter leaky=true name=cutter'\ - ' ! tee name=jsgf ! queue leaky=downstream ! valve name=valve_jsgf drop=true ! pocketsphinx name=asr_jsgf ! fakesink async=false jsgf.'\ - ' ! pocketsphinx name=asr_kws ! fakesink async=false'\ - ) - # Ignore everything below the configured volume - cutter = @pipeline.get_by_name('cutter') - cutter.set_property('threshold-dB', -20) - cutter.set_property('pre-length', 100000000) # pocketsphinx needs about 0.1s before start - cutter.set_property('run-length', 1300000000) - - asr_jsgf = @pipeline.get_by_name('asr_jsgf') - asr_jsgf.set_property('hmm', 'pocketsphinx/adapt/cmusphinx-en-us-5.2') - asr_jsgf.set_property('mllr', 'pocketsphinx/adapt/mllr_matrix') - asr_jsgf.set_property('jsgf', 'data/robot.jsgf') - - asr_kws = @pipeline.get_by_name('asr_kws') - asr_kws.set_property('hmm', 'pocketsphinx/adapt/cmusphinx-en-us-5.2') - asr_kws.set_property('mllr', 'pocketsphinx/adapt/mllr_matrix') - asr_kws.set_property('kws', 'data/keywords.kws') - - bus = @pipeline.bus() - bus.add_watch do |bus, message| - case message.type - when Gst::MessageType::EOS - loop.quit - when Gst::MessageType::ERROR - p message.parse_error - binding.pry # open console - loop.quit - when Gst::MessageType::ELEMENT - if message.src.name == "asr_kws" - if message.structure.get_value(:final).value - keyword_detect(message.structure.get_value(:hypothesis).value, message.structure.get_value(:confidence).value) - end - elsif message.src.name == "asr_jsgf" - if message.structure.get_value(:final).value - final_result(message.structure.get_value(:hypothesis).value, message.structure.get_value(:confidence).value) - end - elsif message.src.name == "cutter" - if message.structure.get_value(:above).value - @logger.debug "Start recording.." - else - @logger.debug "Stop recording" - end - end - end - true - end - - @pipeline.play - end - - # Enables/Disables the jsgf pipeline branch - def enable_jsgf(bEnable) - valve = @pipeline.get_by_name('valve_jsgf') - valve.set_property("drop", !bEnable) - end - - # Result of jsgf pipeline branch - def final_result(hyp, confidence) - @logger.info "final: " + hyp + " " + confidence.to_s - enable_jsgf(false) - - # Publish pocketsphinx result as ros message - msg = Std_msgs::String.new - msg.data = hyp - @publisher.publish(msg) - end - - def keyword_detect(hyp, confidence) - @logger.debug "Got keyword: " + hyp - enable_jsgf(true) - end - - def stop - @pipeline.stop - end -end - -if __FILE__ == $0 - node = ROS::Node.new('pocketsphinx') - app = Speak.new(node) - loop = GLib::MainLoop.new(nil, false) - begin - Thread.new { - loop.run - } - node.spin - rescue Interrupt - ensure - app.stop - node.shutdown - loop.quit - end -end diff --git a/scripts/sensor_board.py b/scripts/sensor_board.py index c5e84c4..c8a0c0c 100755 --- a/scripts/sensor_board.py +++ b/scripts/sensor_board.py @@ -16,6 +16,8 @@ from wild_thumper.msg import Sensor # Board warming offset TEMP_ERROR = -3.0 # -5 # degree celsius PRESSURE_ERROR = -2.5 +A_dew_point = 17.271 +B_dew_point = 237.7 """ LDR: @@ -93,6 +95,8 @@ class SensorBoard: msg.pressure = pressure msg.co = co msg.ventilate = True if ventilate > 1.10 else False + tmp = ((A_dew_point * temp) / (B_dew_point + temp)) + log(humidity/100.0) + msg.dew_point = (B_dew_point * tmp) / (A_dew_point - tmp) self.pub.publish(msg)