\node[minimum width=1.2cm, minimum height=1.2cm,dashed, text=black, draw=black, text width=1.2cm, align=center, rounded corners=1mm, outer sep=0](#3) at (#1) {#2};
}
\tikzexternalenable
- \begin{tikzpicture}
+ \begin{tikzpicture}[x=1mm, y=1mm]
\scriptsize
- \begin{scope}[x=1mm, y=1mm]
+ %\node[anchor=south west] () at (0, 55) {Device};
+ %\draw[very thick] (0,55) -- (170,55);
- %\node[anchor=south west] () at (0, 55) {Device};
- %\draw[very thick] (0,55) -- (170,55);
+ \redcell{15,65}{User action}{}
+ \dashedcell{30,65}{Device HID sent}{mousehid}
+ \darkbluecell{125,65}{Sytem response}{}
- \redcell{15,65}{User action}{}
- \dashedcell{30,65}{Device HID sent}{mousehid}
- \darkbluecell{125,65}{Sytem response}{}
+ \draw[fill=black!10, rounded corners=2mm] (0,0) rectangle (170,55);
- \draw[fill=black!10, rounded corners=2mm] (0,0) rectangle (170,55);
+ \node[anchor=south west] () at (0, 10) {Host computer};
+ \draw[very thick] (0,10) -- (170,10);
- \node[anchor=south west] () at (0, 10) {Host computer};
- \draw[very thick] (0,10) -- (170,10);
+ \bluecell{45,20}{Raw HID received}{rawhidreceived1}
+ \draw[ultra thick] (45,8) -- (45,12);
+ \node[anchor=north] () at (45, 8) {piezo};
+ \dashedcell{60,20}{Device HID received}{hosthid}
+ \draw[ultra thick] (60,8) -- (60,12);
+ \node[anchor=north] () at (60, 8) {libpointing};
+ \dashedcell{75,20}{Low-level event}{}
+ \draw[ultra thick] (75,8) -- (75,12);
+ \node[anchor=north] () at (75, 8) {system};
+ \dashedcell{90,20}{Event delivered}{}
+ \draw[ultra thick] (90,8) -- (90,12);
+ \node[anchor=north] () at (90, 8) {toolkit};
+ \dashedcell{105,20}{Window repaint}{}
+ \draw[ultra thick] (105,8) -- (105,12);
+ \node[anchor=north] () at (105, 8) {repaint};
+ \dashedcell{120,20}{Screen update}{}
+ \draw[ultra thick] (120,8) -- (120,12);
+ \node[anchor=north] () at (120, 8) {screen};
+ \darkbluecell{160,20}{Raw HID received}{rawhidreceived2}
+ \draw[ultra thick] (160,8) -- (160,12);
+ \node[anchor=north] () at (160, 8) {photodiode};
- \bluecell{45,20}{Raw HID received}{rawhidreceived1}
- \draw[ultra thick] (45,8) -- (45,12);
- \node[anchor=north] () at (45, 8) {piezo};
- \dashedcell{60,20}{Device HID received}{hosthid}
- \draw[ultra thick] (60,8) -- (60,12);
- \node[anchor=north] () at (60, 8) {libpointing};
- \dashedcell{75,20}{Low-level event}{}
- \draw[ultra thick] (75,8) -- (75,12);
- \node[anchor=north] () at (75, 8) {system};
- \dashedcell{90,20}{Event delivered}{}
- \draw[ultra thick] (90,8) -- (90,12);
- \node[anchor=north] () at (90, 8) {toolkit};
- \dashedcell{105,20}{Window repaint}{}
- \draw[ultra thick] (105,8) -- (105,12);
- \node[anchor=north] () at (105, 8) {repaint};
- \dashedcell{120,20}{Screen update}{}
- \draw[ultra thick] (120,8) -- (120,12);
- \node[anchor=north] () at (120, 8) {screen};
- \darkbluecell{160,20}{Raw HID received}{rawhidreceived2}
- \draw[ultra thick] (160,8) -- (160,12);
- \node[anchor=north] () at (160, 8) {photodiode};
+ \node[anchor=south west] () at (0, 35) {Arduino};
+ \draw[very thick] (0,35) -- (170,35);
- \node[anchor=south west] () at (0, 35) {Arduino};
- \draw[very thick] (0,35) -- (170,35);
+ \bluecell{20,45}{Piezo trigger}{}
+ \draw[ultra thick] (20,33) -- (20,37);
+ \node[anchor=north] () at (20, 33) {piezo};
+ \bluecell{35,45}{Raw HID sent}{rawhidsent1}
+ \darkbluecell{135,45}{Photo-diode trigger}{}
+ \draw[ultra thick] (135,33) -- (135,37);
+ \node[anchor=north] () at (135, 33) {photodiode};
+ \darkbluecell{150,45}{Raw HID sent}{rawhidsent2}
- \bluecell{20,45}{Piezo trigger}{}
- \draw[ultra thick] (20,33) -- (20,37);
- \node[anchor=north] () at (20, 33) {piezo};
- \bluecell{35,45}{Raw HID sent}{rawhidsent1}
- \darkbluecell{135,45}{Photo-diode trigger}{}
- \draw[ultra thick] (135,33) -- (135,37);
- \node[anchor=north] () at (135, 33) {photodiode};
- \darkbluecell{150,45}{Raw HID sent}{rawhidsent2}
-
- \path[-{Stealth[length=2mm, width=1.5mm]}, dashed, to path={-| (\tikztotarget)}] (mousehid) edge (hosthid);
- \path[-{Stealth[length=2mm, width=1.5mm]}, dashed, to path={-| (\tikztotarget)}] (rawhidsent1) edge (rawhidreceived1);
- \path[-{Stealth[length=2mm, width=1.5mm]}, dashed, to path={-| (\tikztotarget)}] (rawhidsent2) edge (rawhidreceived2);
-
- \end{scope}
+ \path[-{Stealth[length=2mm, width=1.5mm]}, dashed, to path={-| (\tikztotarget)}] (mousehid) edge (hosthid);
+ \path[-{Stealth[length=2mm, width=1.5mm]}, dashed, to path={-| (\tikztotarget)}] (rawhidsent1) edge (rawhidreceived1);
+ \path[-{Stealth[length=2mm, width=1.5mm]}, dashed, to path={-| (\tikztotarget)}] (rawhidsent2) edge (rawhidreceived2);
\end{tikzpicture}
\tikzexternaldisable
\caption[LagMeter measurement process.]{LagMeter measurement process.}
--- /dev/null
+%!TEX root = ../hdrmain.tex
+
+\begin{figure}[htb]
+ \definecolor{customgreen}{RGB} {21,156,62}
+ \definecolor{customred}{RGB} {244,8,39}
+ \definecolor{customblue}{RGB} {16 0 255}
+
+ \tikzexternalenable
+ \begin{tikzpicture}[x=1mm,y=1mm]
+ \small
+ \coordinate (threshold) at (0,45);
+ \coordinate (fingerpoint) at (28.8,26);
+ \coordinate (buttonpoint) at (56.1,27);
+ \coordinate (piezopoint) at (83,45);
+ \coordinate (fingerbottom) at ($(fingerpoint) - (0,13)$);
+ \coordinate (buttonbottom) at ($(buttonpoint) - (0,14)$);
+ \coordinate (piezobottom) at ($(piezopoint) - (0,32)$);
+ %
+ \node[anchor=south west,inner sep=0,outer sep=0] (oscillo) {
+ \includegraphics[width=\columnwidth]{figures/piezotrigger.png}
+ };
+ %\draw[help lines] (0,0) grid (170,80);
+ \node[customblue, anchor=north] (finger) at (fingerbottom) {Finger contact};
+ \node[customred, anchor=north] (button) at (buttonbottom) {Button pressed};
+ \node[customgreen, anchor=north] (piezo) at (piezobottom) {Piezo trigger};
+
+ % Threshold
+
+ \draw[draw=violet, ultra thick] (threshold) to ($(threshold) + (169,0)$);
+ \node[violet, anchor=south east] () at ($(threshold) + (160,0)$) {$1.5V$ Threshold};
+
+ % Arrows
+ \draw[draw=customblue, -stealth', very thick, anchor=north] (fingerbottom) to (fingerpoint);
+ \draw[draw=customred, -stealth', very thick] (buttonbottom) to (buttonpoint);
+ \draw[draw=customgreen, -stealth', very thick] (piezobottom) to (piezopoint);
+ \end{tikzpicture}
+ \tikzexternaldisable
+ \label{fig:piezotrigger}
+ \caption[Latency measurement of a button press]{Latency measurement of a button press. The blue line goes to 5V when the finger touches the surface. The red lines goes to 0V when the button is pressed. The green curves shows the piezoelectric vibratio sensor signal. The violet line shows a $1.5V$ threshold.}
+\end{figure}
We designed LagMeter, a latency measurement tool that facilitates repetitive measures, and enables the slicing of latency between several parts of the interactive system~\cite{casiez17}.
It measures latency for any tap-based input such as keyboards, mice, touchpads, touchscreens.
-The slicing of latency uses software probes at different stages of the system.
+%The slicing of latency uses software probes at different stages of the system.
The details are described in the paper, and the overall idea is depicted on \reffig{fig:lagmeter}.
-
-The moment users touch the input device, a piezoelectric vibration sensor\footnote{\href{http://www.te.com/usa-en/product-CAT-PFS0011.html}{http://www.te.com/usa-en/product-CAT- PFS0011.html}} detects the contact.
-We connected this sensor to a custom board connected to an Arduino Leonardo\footnote{\href{https://www.arduino.cc/en/Main/Arduino_BoardLeonardo}{https://www.arduino.cc/en/Main/Arduino\_BoardLeonardo}} microcontroller board, which sends a Raw HID message to the host computer.
-This communication channel is fast, we measured a $1.3s$ round-trip delay between the Arduino and the host computer (sd $0.6s$, \nth{95} percentile $2.0s$, details are available in the paper).
-In parallel, the input device sends an HID packet corresponding to the event to the host computer.
+The light blue items correspond to input stages and dark blue items to output stages.
+The dashed items are optional steps that provide finer grain information for latency slicing.
+They depend on the system running on the host computer.
+
+The moment users touch the input device, a piezoelectric vibration sensor\footnote{\href{http://www.te.com/usa-en/product-CAT-PFS0011.html}{http://www.te.com/usa-en/product-CAT- PFS0011.html}} attached to the fingertip detects the contact.
+The idea is that all touch-based input require an initial contact.
+We connected this sensor to a custom electronic board connected to an Arduino Leonardo\footnote{\href{https://www.arduino.cc/en/Main/Arduino_BoardLeonardo}{https://www.arduino.cc/en/Main/Arduino\_BoardLeonardo}} microcontroller board.
+The board enables adjustmenting the detection threshold since the voltage we get from the piezo sensor is typically lower than $0.5V$.
+The Arduino sends a Raw HID message to the host computer when it detects contacts.
+In parallel, the input device sends an event HID packet to the host computer.
A custom application gets the RawHID packet with HIDAPI\footnote{\href{https://github.com/signal11/hidapi}{https://github.com/signal11/hidapi}}, and the device HID packet with libpointing~\cite{casiez11a}.
-It also has callbacks on the system and toolkit input events.
+It also has callbacks on the system and the toolkit input events.
When the application received the input event, the application toggles the screen between black and white.
-It has callbacks that trigger when the repaint is requested and done.
-Then, a photodiode on the electronic board detects the change on the screen and the Arduino sends another Raw HID message to the application.
+It has callbacks that trigger when the screen repaint is requested and done.
+Then, a photodiode on the electronic board detects changes on the screen and the Arduino sends another Raw HID message to the application.
%Our method is usable with any tap-based input: keyboard, mouse, touchpad, touchscreen.
%Slicing of latency with software probes.
\paragraph{Results and discussion}
-briefly describe results, takeaway: latency is due do display pipeline
+What I realized with this project is that nothing in interaction can be considered as instantaneous.
+When we measured the responsiveness of the piezoelectric vibration sensor, we noticed that even the mechanical contact of a physical button creates latency at the millisecond scale.
+It is both due to the elasticity of the finger, and the mechanical actuation of the button switch.
+We made this measurement with an aluminium foil around a finger, tapping on a copper tape pasted on a mouse button.
+The finger also had a piezoelectric sensor strapped around its tip.
+The \reffig{fig:piezotrigger} shows a measurement with three signals: the finger contact in blue, the button press in red and the piezo signal in green.
+We notice it takes about $2ms$ for the button to switch.
+The delay of the piezo trigger depends on the threshold, but it typically takes between $1ms$ and $3ms$.
+The paper contains more detail data about these measures~\cite{casiez17}.
+
+\input{figures/piezotrigger.tex}
+
+The measurement of the screen change was more accurate.
+The idea was to switch the screen between black and white so that we can detect the change with a simple luminosity sensor.
+We used a photodiode because it is fast.
+The \reffig{fig:flicker} shows the signal when the screen turned white.
+Interestingly, the measure is so fast that we can observe the lines being drawn around the sensor.
+Thereforer the position of the photodiode on the screen has an impact on the latency we measure.
+However, depending on the adjustment of the detection threshold, we can reliably measure the screen change in $4\mu s$.
+This is way sufficient for our purpose, and makes the actual position of the sensor less critical.
+
+\begin{figure}[htb]
+ \includegraphics[width=\columnwidth]{figures/screenflicker}
+ \label{fig:flicker}
+ \caption[Photodiode measure]{Photodiode measure. The green signal is the photodiode signal and the red signal is the amplified signal.}
+\end{figure}
+
+Latency slicing requires having all measures with the same clock.
+This was an issue because part of the process was done on the Arduino board, and another part on the host computer.
+It means we needed a way to communicate in a fast and reliable way between the two.
+The problem is that typical operating systems (Windows, Linux and MacOS) cannot guarantee the execution of code with a $1ms$ precision.
+One solution was to use a Raspberry Pi\footnote{\href{https://www.raspberrypi.org/}{https://www.raspberrypi.org/}} as a host computer because it has GPIOs that can communicate with microcontrollers with low latency.
+However, measures with this platform would not necessarily be representative to the usage of other platforms.
+Thus we considered several communication channels between a microcontroller and a host computer such as ethernet or a parallel port with a specific extension card.
+Nevertheless, we opted for raw HID messages because its latency was sufficiently low and consistent.
+The round-trip delay between the Arduino and the host computer was between $1s$ and $2s$, depending on the operating system and the processor load.
+%Details are available in the paper.
+
+We made multiple series of measures to study the impact of many types of factors.
+Concerning the input, we compared different input frequencies with a Logitech G9 mouse that enables the adjustment of the mouse frequency.
+%: $125Hz$, $250Hz$, $500Hz$ and $1000Hz$.
+%The difference of latency is small.
+The host computer received the device HID packet after $3.3ms$ in average for $125Hz$ and under a millisecond for frequencies above $250Hz$.
+The latency between the moment the device HID packet was received and the moment the application requested the repaint of the screen was about $3ms$.
+The end-to-end latency was aroung $60ms$, therefore the input part of the pipeline has little impact on latency.
+%The system and toolkit part are fast as well.
+Most of the latency is due to the output.
+We observed several factors that impact latency: the operating system, graphic toolkit, and screen frequency.
+We used a minimal application for our measures, but real applications most likely introduce more latency because of their normal operations.
+%note: minimal application, more latency with a real applicaiton
+
+%expected latency: hid packet at 1kHz, capacitive measure, event queue, display, repaint
+%Precision of measure, precision of communication between arduino and host, need to have a common clock
+
+%briefly describe results, takeaway: latency is due do display pipeline
+
+%Output imposes constraints on input
+%subpixel\cite{roussel12}
+%hands dexterity \cite{aceituno13}
\subsection{Flexible pens}
\label{sec:flexiblepens}
FlexStylus \cite{fellion17}
Hyperbrush \cite{guerrero21}
+
\subsection{Finger identification}
\label{sec:fingeridentification}
Leverages finger identification for command selection and direct manipulation of parameters.
+
\subsection{Interaction in VR}
\label{sec:interactionvr}
Facial expressions \cite{baloup21}
+
\section{Conclusion}
projects / hdr.git / commitdiff
