[AdvancedPaste]Improved telemetry (#33520)

<!-- Enter a brief description/summary of your PR here. What does it
fix/what does it change/how was it tested (even manually, if necessary)?
-->
## Summary of the Pull Request

This PR improves advanced paste telemetry. Here's what's changed

- Added `AdvancedPasteClipboardItemClicked` event
- Changed `CustomFormatEvent` to only fire on successful completion and
include the number of tokens used, and the model name

Here are the goals of adding this telemtry:

- `AdvancedPasteClipboardItemClicked` helps us estimate the total number
of user who are using the clipboard history feature, which helps us
prioritize future investments and improvements in PowerToys. (This is
just regular feature usage data).
- `CustomFormatEvent` now includes number of tokens used, and the model
name. We are considering using alternative models to power Advanced
Paste (as we've heard feedback about this), and these different models
could have different token lengths. Understanding the average usage will
help us make good decisions that will benefit the most users. As well,
the model name is hard coded right now, but in the future if we do add
different AI models for completion then this will help us compare their
performance.

<!-- Please review the items on the PR checklist before submitting-->
## PR Checklist

- [X] Communication: I've discussed this with core contributors already.
If work hasn't been agreed, this work might be rejected


<!-- Provide a more detailed description of the PR, other things fixed
or any additional comments/features here -->
## Detailed Description of the Pull Request / Additional comments

The details above are detailed enough since this change is super small. 

<!-- Describe how you validated the behavior. Add automated tests
wherever possible, but list manual validation steps taken as well -->
## Validation Steps Performed

Ensured that PowerToys successfully built (Need help verifying the
events fire off correctly).
This commit is contained in:
Craig Loewen 2024-06-27 11:42:15 -04:00 committed by GitHub
parent 8b8c75b9a5
commit 3b7adbe6ac
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 42 additions and 6 deletions

View File

@ -78,8 +78,6 @@ namespace AdvancedPaste.Controls
{
Logger.LogTrace();
PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteGenerateCustomFormatEvent());
VisualStateManager.GoToState(this, "LoadingState", true);
string inputInstructions = InputTxtBox.Text;
ViewModel.SaveQuery(inputInstructions);

View File

@ -231,6 +231,7 @@ namespace AdvancedPaste.Pages
var item = e.ClickedItem as ClipboardItem;
if (item is not null)
{
PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteClipboardItemClicked());
if (!string.IsNullOrEmpty(item.Content))
{
ClipboardHelper.SetClipboardTextContent(item.Content);

View File

@ -33,6 +33,8 @@ namespace AdvancedPaste.Helpers
private string _openAIKey;
private string _modelName = "gpt-3.5-turbo-instruct";
public bool IsAIEnabled => !string.IsNullOrEmpty(this._openAIKey);
public AICompletionsHelper()
@ -69,14 +71,14 @@ namespace AdvancedPaste.Helpers
return string.Empty;
}
public string GetAICompletion(string systemInstructions, string userMessage)
private Response<Completions> GetAICompletion(string systemInstructions, string userMessage)
{
OpenAIClient azureAIClient = new OpenAIClient(_openAIKey);
var response = azureAIClient.GetCompletions(
new CompletionsOptions()
{
DeploymentName = "gpt-3.5-turbo-instruct",
DeploymentName = _modelName,
Prompts =
{
systemInstructions + "\n\n" + userMessage,
@ -90,7 +92,7 @@ namespace AdvancedPaste.Helpers
Console.WriteLine("Cut off due to length constraints");
}
return response.Value.Choices[0].Text;
return response;
}
public AICompletionsResponse AIFormatString(string inputInstructions, string inputString)
@ -109,10 +111,16 @@ Output:
";
string aiResponse = null;
Response<Completions> rawAIResponse = null;
int apiRequestStatus = (int)HttpStatusCode.OK;
try
{
aiResponse = this.GetAICompletion(systemInstructions, userMessage);
rawAIResponse = this.GetAICompletion(systemInstructions, userMessage);
aiResponse = rawAIResponse.Value.Choices[0].Text;
int promptTokens = rawAIResponse.Value.Usage.PromptTokens;
int completionTokens = rawAIResponse.Value.Usage.CompletionTokens;
PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteGenerateCustomFormatEvent(promptTokens, completionTokens, _modelName));
}
catch (Azure.RequestFailedException error)
{

View File

@ -0,0 +1,16 @@
// Copyright (c) Microsoft Corporation
// The Microsoft Corporation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System.Diagnostics.Tracing;
using Microsoft.PowerToys.Telemetry;
using Microsoft.PowerToys.Telemetry.Events;
namespace AdvancedPaste.Telemetry
{
[EventData]
public class AdvancedPasteClipboardItemClicked : EventBase, IEvent
{
public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage;
}
}

View File

@ -11,6 +11,19 @@ namespace AdvancedPaste.Telemetry
[EventData]
public class AdvancedPasteGenerateCustomFormatEvent : EventBase, IEvent
{
public int PromptTokens { get; set; }
public int CompletionTokens { get; set; }
public string ModelName { get; set; }
public AdvancedPasteGenerateCustomFormatEvent(int promptTokens, int completionTokens, string modelName)
{
this.PromptTokens = promptTokens;
this.CompletionTokens = completionTokens;
ModelName = modelName;
}
public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage;
}
}